Persist "tagged" network stats along with UIDs.

Now stores tags other than TAG_NONE (0x0), which are useful for app
debugging.  Combine UID and tag together into single long key, and
expose tag data through AIDL when requested.  Change NMS to track
TAG_NONE as total UID traffic, matching the kernel definition.

Added TAG_MAX_HISTORY to control how long tag-granularity statistics
are stored; overall UID usage is still kept for UID_MAX_HISTORY.  Fix
bug to trim NetworkStatsHistory outside normal polling loops to catch
non-active networks and UIDs.

Test to verify UID and tag packing, and to verify that UID traffic on
two networks are combined to match MOBILE_ALL template.

Change-Id: If0e039416d9e7f63b1a39e04cddfb1133b5a78ee

services/tests/servicestests/src/com/android/server/NetworkStatsServiceTest.java

@@ -21,6 +21,8 @@ import static android.content.Intent.EXTRA_UID;
 import static android.net.ConnectivityManager.CONNECTIVITY_ACTION;
 import static android.net.ConnectivityManager.TYPE_MOBILE;
 import static android.net.ConnectivityManager.TYPE_WIFI;
+import static android.net.ConnectivityManager.TYPE_WIMAX;
+import static android.net.NetworkStats.IFACE_ALL;
 import static android.net.NetworkStats.TAG_NONE;
 import static android.net.NetworkStats.UID_ALL;
 import static android.net.NetworkTemplate.MATCH_MOBILE_ALL;
@@ -31,6 +33,9 @@ import static android.text.format.DateUtils.HOUR_IN_MILLIS;
 import static android.text.format.DateUtils.MINUTE_IN_MILLIS;
 import static android.text.format.DateUtils.WEEK_IN_MILLIS;
 import static com.android.server.net.NetworkStatsService.ACTION_NETWORK_STATS_POLL;
+import static com.android.server.net.NetworkStatsService.packUidAndTag;
+import static com.android.server.net.NetworkStatsService.unpackTag;
+import static com.android.server.net.NetworkStatsService.unpackUid;
 import static org.easymock.EasyMock.anyLong;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.eq;
@@ -54,6 +59,7 @@ import android.os.INetworkManagementService;
 import android.telephony.TelephonyManager;
 import android.test.AndroidTestCase;
 import android.test.suitebuilder.annotation.LargeTest;
+import android.util.Log;
 import android.util.TrustedTime;
 
 import com.android.server.net.NetworkStatsService;
@@ -80,9 +86,9 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
     private static NetworkTemplate sTemplateImsi1 = new NetworkTemplate(MATCH_MOBILE_ALL, IMSI_1);
     private static NetworkTemplate sTemplateImsi2 = new NetworkTemplate(MATCH_MOBILE_ALL, IMSI_2);
 
-    private static final int TEST_UID_RED = 1001;
+    private static final int UID_RED = 1001;
-    private static final int TEST_UID_BLUE = 1002;
+    private static final int UID_BLUE = 1002;
-    private static final int TEST_UID_GREEN = 1003;
+    private static final int UID_GREEN = 1003;
 
     private BroadcastInterceptingContext mServiceContext;
     private File mStatsDir;
@@ -216,16 +222,16 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expectNetworkStatsSummary(new NetworkStats(elapsedRealtime, 1)
                 .addEntry(TEST_IFACE, UID_ALL, TAG_NONE, 1024L, 2048L));
         expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 2)
-                .addEntry(TEST_IFACE, TEST_UID_RED, TAG_NONE, 512L, 256L)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 512L, 256L)
-                .addEntry(TEST_IFACE, TEST_UID_BLUE, TAG_NONE, 128L, 128L));
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 128L, 128L));
 
         replay();
         mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
 
         // verify service recorded history
         assertNetworkTotal(sTemplateWifi, 1024L, 2048L);
-        assertUidTotal(sTemplateWifi, TEST_UID_RED, 512L, 256L);
+        assertUidTotal(sTemplateWifi, UID_RED, 512L, 256L);
-        assertUidTotal(sTemplateWifi, TEST_UID_BLUE, 128L, 128L);
+        assertUidTotal(sTemplateWifi, UID_BLUE, 128L, 128L);
         verifyAndReset();
 
         // graceful shutdown system, which should trigger persist of stats, and
@@ -250,8 +256,8 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
 
         // after systemReady(), we should have historical stats loaded again
         assertNetworkTotal(sTemplateWifi, 1024L, 2048L);
-        assertUidTotal(sTemplateWifi, TEST_UID_RED, 512L, 256L);
+        assertUidTotal(sTemplateWifi, UID_RED, 512L, 256L);
-        assertUidTotal(sTemplateWifi, TEST_UID_BLUE, 128L, 128L);
+        assertUidTotal(sTemplateWifi, UID_BLUE, 128L, 128L);
         verifyAndReset();
 
     }
@@ -335,9 +341,9 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expectNetworkStatsSummary(new NetworkStats(elapsedRealtime, 1)
                 .addEntry(TEST_IFACE, UID_ALL, TAG_NONE, 2048L, 512L));
         expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 3)
-                .addEntry(TEST_IFACE, TEST_UID_RED, TAG_NONE, 1024L, 0L)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 1536L, 512L)
-                .addEntry(TEST_IFACE, TEST_UID_RED, 0xF00D, 512L, 512L)
+                .addEntry(TEST_IFACE, UID_RED, 0xF00D, 512L, 512L)
-                .addEntry(TEST_IFACE, TEST_UID_BLUE, TAG_NONE, 512L, 0L));
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 512L, 0L));
 
         replay();
         mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
@@ -345,8 +351,8 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         // verify service recorded history
         assertNetworkTotal(sTemplateImsi1, 2048L, 512L);
         assertNetworkTotal(sTemplateWifi, 0L, 0L);
-        assertUidTotal(sTemplateImsi1, TEST_UID_RED, 1536L, 512L);
+        assertUidTotal(sTemplateImsi1, UID_RED, 1536L, 512L);
-        assertUidTotal(sTemplateImsi1, TEST_UID_BLUE, 512L, 0L);
+        assertUidTotal(sTemplateImsi1, UID_BLUE, 512L, 0L);
         verifyAndReset();
 
         // now switch networks; this also tests that we're okay with interfaces
@@ -370,21 +376,21 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expectNetworkStatsSummary(new NetworkStats(elapsedRealtime, 1)
                 .addEntry(TEST_IFACE, UID_ALL, TAG_NONE, 128L, 1024L));
         expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
-                .addEntry(TEST_IFACE, TEST_UID_BLUE, TAG_NONE, 128L, 1024L));
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 128L, 1024L));
 
         replay();
         mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
 
         // verify original history still intact
         assertNetworkTotal(sTemplateImsi1, 2048L, 512L);
-        assertUidTotal(sTemplateImsi1, TEST_UID_RED, 1536L, 512L);
+        assertUidTotal(sTemplateImsi1, UID_RED, 1536L, 512L);
-        assertUidTotal(sTemplateImsi1, TEST_UID_BLUE, 512L, 0L);
+        assertUidTotal(sTemplateImsi1, UID_BLUE, 512L, 0L);
 
         // and verify new history also recorded under different template, which
         // verifies that we didn't cross the streams.
         assertNetworkTotal(sTemplateImsi2, 128L, 1024L);
         assertNetworkTotal(sTemplateWifi, 0L, 0L);
-        assertUidTotal(sTemplateImsi2, TEST_UID_BLUE, 128L, 1024L);
+        assertUidTotal(sTemplateImsi2, UID_BLUE, 128L, 1024L);
         verifyAndReset();
 
     }
@@ -409,18 +415,18 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expectNetworkStatsSummary(new NetworkStats(elapsedRealtime, 1)
                 .addEntry(TEST_IFACE, UID_ALL, TAG_NONE, 4128L, 544L));
         expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
-                .addEntry(TEST_IFACE, TEST_UID_RED, TAG_NONE, 16L, 16L)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 16L, 16L)
-                .addEntry(TEST_IFACE, TEST_UID_BLUE, TAG_NONE, 4096L, 512L)
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 4096L, 512L)
-                .addEntry(TEST_IFACE, TEST_UID_GREEN, TAG_NONE, 16L, 16L));
+                .addEntry(TEST_IFACE, UID_GREEN, TAG_NONE, 16L, 16L));
 
         replay();
         mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
 
         // verify service recorded history
         assertNetworkTotal(sTemplateWifi, 4128L, 544L);
-        assertUidTotal(sTemplateWifi, TEST_UID_RED, 16L, 16L);
+        assertUidTotal(sTemplateWifi, UID_RED, 16L, 16L);
-        assertUidTotal(sTemplateWifi, TEST_UID_BLUE, 4096L, 512L);
+        assertUidTotal(sTemplateWifi, UID_BLUE, 4096L, 512L);
-        assertUidTotal(sTemplateWifi, TEST_UID_GREEN, 16L, 16L);
+        assertUidTotal(sTemplateWifi, UID_GREEN, 16L, 16L);
         verifyAndReset();
 
         // now pretend two UIDs are uninstalled, which should migrate stats to
@@ -428,22 +434,162 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expectDefaultSettings();
         replay();
         final Intent intent = new Intent(ACTION_UID_REMOVED);
-        intent.putExtra(EXTRA_UID, TEST_UID_BLUE);
+        intent.putExtra(EXTRA_UID, UID_BLUE);
         mServiceContext.sendBroadcast(intent);
-        intent.putExtra(EXTRA_UID, TEST_UID_RED);
+        intent.putExtra(EXTRA_UID, UID_RED);
         mServiceContext.sendBroadcast(intent);
 
         // existing uid and total should remain unchanged; but removed UID
         // should be gone completely.
         assertNetworkTotal(sTemplateWifi, 4128L, 544L);
-        assertUidTotal(sTemplateWifi, TEST_UID_RED, 0L, 0L);
+        assertUidTotal(sTemplateWifi, UID_RED, 0L, 0L);
-        assertUidTotal(sTemplateWifi, TEST_UID_BLUE, 0L, 0L);
+        assertUidTotal(sTemplateWifi, UID_BLUE, 0L, 0L);
-        assertUidTotal(sTemplateWifi, TEST_UID_GREEN, 16L, 16L);
+        assertUidTotal(sTemplateWifi, UID_GREEN, 16L, 16L);
         assertUidTotal(sTemplateWifi, UID_REMOVED, 4112L, 528L);
         verifyAndReset();
 
     }
 
+    public void testUid3g4gCombinedByTemplate() throws Exception {
+        long elapsedRealtime = 0;
+
+        // pretend that network comes online
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkState(buildMobile3gState(IMSI_1));
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(CONNECTIVITY_ACTION));
+        verifyAndReset();
+
+        // create some traffic
+        elapsedRealtime += HOUR_IN_MILLIS;
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+        expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 1024L, 1024L)
+                .addEntry(TEST_IFACE, UID_RED, 0xF00D, 512L, 512L));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
+
+        // verify service recorded history
+        assertUidTotal(sTemplateImsi1, UID_RED, 1024L, 1024L);
+        verifyAndReset();
+
+        // now switch over to 4g network
+        elapsedRealtime += HOUR_IN_MILLIS;
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkState(buildMobile4gState());
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+        expectNetworkStatsDetail(buildEmptyStats(elapsedRealtime));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(CONNECTIVITY_ACTION));
+        mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
+        verifyAndReset();
+
+        // create traffic on second network
+        elapsedRealtime += HOUR_IN_MILLIS;
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+        expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 512L, 256L));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
+
+        // verify that ALL_MOBILE template combines both
+        assertUidTotal(sTemplateImsi1, UID_RED, 1536L, 1280L);
+
+        verifyAndReset();
+
+    }
+    
+    public void testPackedUidAndTag() throws Exception {
+        assertEquals(0x0000000000000000L, packUidAndTag(0, 0x0));
+        assertEquals(0x000003E900000000L, packUidAndTag(1001, 0x0));
+        assertEquals(0x000003E90000F00DL, packUidAndTag(1001, 0xF00D));
+
+        long packed;
+        packed = packUidAndTag(Integer.MAX_VALUE, Integer.MIN_VALUE);
+        assertEquals(Integer.MAX_VALUE, unpackUid(packed));
+        assertEquals(Integer.MIN_VALUE, unpackTag(packed));
+
+        packed = packUidAndTag(Integer.MIN_VALUE, Integer.MAX_VALUE);
+        assertEquals(Integer.MIN_VALUE, unpackUid(packed));
+        assertEquals(Integer.MAX_VALUE, unpackTag(packed));
+
+        packed = packUidAndTag(10005, 0xFFFFFFFF);
+        assertEquals(10005, unpackUid(packed));
+        assertEquals(0xFFFFFFFF, unpackTag(packed));
+        
+    }
+
+    public void testSummaryForAllUid() throws Exception {
+        long elapsedRealtime = 0;
+
+        // pretend that network comes online
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkState(buildWifiState());
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(CONNECTIVITY_ACTION));
+        verifyAndReset();
+
+        // create some traffic for two apps
+        elapsedRealtime += HOUR_IN_MILLIS;
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+        expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
+                .addEntry(TEST_IFACE, UID_RED, TAG_NONE, 50L, 50L)
+                .addEntry(TEST_IFACE, UID_RED, 0xF00D, 10L, 10L)
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 1024L, 512L));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
+
+        // verify service recorded history
+        assertUidTotal(sTemplateWifi, UID_RED, 50L, 50L);
+        assertUidTotal(sTemplateWifi, UID_BLUE, 1024L, 512L);
+        verifyAndReset();
+        
+        // now create more traffic in next hour, but only for one app
+        elapsedRealtime += HOUR_IN_MILLIS;
+        expectTime(TEST_START + elapsedRealtime);
+        expectDefaultSettings();
+        expectNetworkStatsSummary(buildEmptyStats(elapsedRealtime));
+        expectNetworkStatsDetail(new NetworkStats(elapsedRealtime, 1)
+                .addEntry(TEST_IFACE, UID_BLUE, TAG_NONE, 2048L, 1024L));
+
+        replay();
+        mServiceContext.sendBroadcast(new Intent(ACTION_NETWORK_STATS_POLL));
+
+        // first verify entire history present
+        NetworkStats stats = mService.getSummaryForAllUid(
+                sTemplateWifi, Long.MIN_VALUE, Long.MAX_VALUE, true);
+        assertEquals(3, stats.size);
+        assertStatsEntry(stats, 0, IFACE_ALL, UID_RED, TAG_NONE, 50L, 50L);
+        assertStatsEntry(stats, 1, IFACE_ALL, UID_RED, 0xF00D, 10L, 10L);
+        assertStatsEntry(stats, 2, IFACE_ALL, UID_BLUE, TAG_NONE, 2048L, 1024L);
+
+        // now verify that recent history only contains one uid
+        final long currentTime = TEST_START + elapsedRealtime;
+        stats = mService.getSummaryForAllUid(
+                sTemplateWifi, currentTime - HOUR_IN_MILLIS, currentTime, true);
+        assertEquals(1, stats.size);
+        assertStatsEntry(stats, 0, IFACE_ALL, UID_BLUE, TAG_NONE, 1024L, 512L);
+
+        verifyAndReset();
+    }
+
     private void assertNetworkTotal(NetworkTemplate template, long rx, long tx) {
         final NetworkStatsHistory history = mService.getHistoryForNetwork(template);
         final long[] total = history.getTotalData(Long.MIN_VALUE, Long.MAX_VALUE, null);
@@ -452,7 +598,7 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
     }
 
     private void assertUidTotal(NetworkTemplate template, int uid, long rx, long tx) {
-        final NetworkStatsHistory history = mService.getHistoryForUid(template, uid);
+        final NetworkStatsHistory history = mService.getHistoryForUid(template, uid, TAG_NONE);
         final long[] total = history.getTotalData(Long.MIN_VALUE, Long.MAX_VALUE, null);
         assertEquals(rx, total[0]);
         assertEquals(tx, total[1]);
@@ -491,6 +637,7 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         expect(mSettings.getNetworkMaxHistory()).andReturn(maxHistory).anyTimes();
         expect(mSettings.getUidBucketDuration()).andReturn(bucketDuration).anyTimes();
         expect(mSettings.getUidMaxHistory()).andReturn(maxHistory).anyTimes();
+        expect(mSettings.getTagMaxHistory()).andReturn(maxHistory).anyTimes();
         expect(mSettings.getTimeCacheMaxAge()).andReturn(DAY_IN_MILLIS).anyTimes();
     }
 
@@ -514,6 +661,15 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         }
     }
 
+    private static void assertStatsEntry(
+            NetworkStats stats, int i, String iface, int uid, int tag, long rx, long tx) {
+        assertEquals(iface, stats.iface[i]);
+        assertEquals(uid, stats.uid[i]);
+        assertEquals(tag, stats.tag[i]);
+        assertEquals(rx, stats.rx[i]);
+        assertEquals(tx, stats.tx[i]);
+    }
+
     private static NetworkState buildWifiState() {
         final NetworkInfo info = new NetworkInfo(TYPE_WIFI, 0, null, null);
         info.setDetailedState(DetailedState.CONNECTED, null, null);
@@ -531,6 +687,14 @@ public class NetworkStatsServiceTest extends AndroidTestCase {
         return new NetworkState(info, prop, null, subscriberId);
     }
 
+    private static NetworkState buildMobile4gState() {
+        final NetworkInfo info = new NetworkInfo(TYPE_WIMAX, 0, null, null);
+        info.setDetailedState(DetailedState.CONNECTED, null, null);
+        final LinkProperties prop = new LinkProperties();
+        prop.setInterfaceName(TEST_IFACE);
+        return new NetworkState(info, prop, null);
+    }
+
     private static NetworkStats buildEmptyStats(long elapsedRealtime) {
         return new NetworkStats(elapsedRealtime, 0);
     }