Transition from DEV network stats to XT.

When XT stats are available, transition to prefer them over DEV,
since they aren't subject to hardware driver bugs.  Only switches at
the first atomic XT bucket, and adds a Settings.Secure flag to force
back to DEV if needed.  Includes tests to cover transition.

Fix tests where device overlay would change which network types
reflected data usage.  Test both history and summary APIs.  Fixed
collection timestamps to reflect full buckets.

Bug: 6504744
Change-Id: Idd7f3b2fdb064c36547c85c51c214fd938c59b7e

core/java/android/net/NetworkStats.java

@@ -111,6 +111,14 @@ public class NetworkStats implements Parcelable {
                     && operations == 0;
         }
 
+        public void add(Entry another) {
+            this.rxBytes += another.rxBytes;
+            this.rxPackets += another.rxPackets;
+            this.txBytes += another.txBytes;
+            this.txPackets += another.txPackets;
+            this.operations += another.operations;
+        }
+
         @Override
         public String toString() {
             final StringBuilder builder = new StringBuilder();

core/java/android/net/NetworkStatsHistory.java

@@ -342,11 +342,23 @@ public class NetworkStatsHistory implements Parcelable {
      * for combining together stats for external reporting.
      */
     public void recordEntireHistory(NetworkStatsHistory input) {
+        recordHistory(input, Long.MIN_VALUE, Long.MAX_VALUE);
+    }
+
+    /**
+     * Record given {@link NetworkStatsHistory} into this history, copying only
+     * buckets that atomically occur in the inclusive time range. Doesn't
+     * interpolate across partial buckets.
+     */
+    public void recordHistory(NetworkStatsHistory input, long start, long end) {
         final NetworkStats.Entry entry = new NetworkStats.Entry(
                 IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
         for (int i = 0; i < input.bucketCount; i++) {
-            final long start = input.bucketStart[i];
+            final long bucketStart = input.bucketStart[i];
-            final long end = start + input.bucketDuration;
+            final long bucketEnd = bucketStart + input.bucketDuration;
+
+            // skip when bucket is outside requested range
+            if (bucketStart < start || bucketEnd > end) continue;
 
             entry.rxBytes = getLong(input.rxBytes, i, 0L);
             entry.rxPackets = getLong(input.rxPackets, i, 0L);
@@ -354,7 +366,7 @@ public class NetworkStatsHistory implements Parcelable {
             entry.txPackets = getLong(input.txPackets, i, 0L);
             entry.operations = getLong(input.operations, i, 0L);
 
-            recordData(start, end, entry);
+            recordData(bucketStart, bucketEnd, entry);
         }
     }
 

core/java/android/net/NetworkTemplate.java

@@ -61,6 +61,13 @@ public class NetworkTemplate implements Parcelable {
                 com.android.internal.R.array.config_data_usage_network_types);
     }
 
+    private static boolean sForceAllNetworkTypes = false;
+
+    // @VisibleForTesting
+    public static void forceAllNetworkTypes() {
+        sForceAllNetworkTypes = true;
+    }
+
     /**
      * Template to match {@link ConnectivityManager#TYPE_MOBILE} networks with
      * the given IMSI.
@@ -225,7 +232,7 @@ public class NetworkTemplate implements Parcelable {
             // TODO: consider matching against WiMAX subscriber identity
             return true;
         } else {
-            return (contains(DATA_USAGE_NETWORK_TYPES, ident.mType)
+            return ((sForceAllNetworkTypes || contains(DATA_USAGE_NETWORK_TYPES, ident.mType))
                     && Objects.equal(mSubscriberId, ident.mSubscriberId));
         }
     }
@@ -291,7 +298,7 @@ public class NetworkTemplate implements Parcelable {
         if (ident.mType == TYPE_WIMAX) {
             return true;
         } else {
-            return contains(DATA_USAGE_NETWORK_TYPES, ident.mType);
+            return sForceAllNetworkTypes || contains(DATA_USAGE_NETWORK_TYPES, ident.mType);
         }
     }
 

services/java/com/android/server/net/NetworkStatsCollection.java

@@ -71,7 +71,7 @@ public class NetworkStatsCollection implements FileRotator.Reader {
 
     private HashMap<Key, NetworkStatsHistory> mStats = Maps.newHashMap();
 
-    private long mBucketDuration;
+    private final long mBucketDuration;
 
     private long mStartMillis;
     private long mEndMillis;
@@ -95,6 +95,18 @@ public class NetworkStatsCollection implements FileRotator.Reader {
         return mStartMillis;
     }
 
+    /**
+     * Return first atomic bucket in this collection, which is more conservative
+     * than {@link #mStartMillis}.
+     */
+    public long getFirstAtomicBucketMillis() {
+        if (mStartMillis == Long.MAX_VALUE) {
+            return Long.MAX_VALUE;
+        } else {
+            return mStartMillis + mBucketDuration;
+        }
+    }
+
     public long getEndMillis() {
         return mEndMillis;
     }
@@ -121,6 +133,15 @@ public class NetworkStatsCollection implements FileRotator.Reader {
      */
     public NetworkStatsHistory getHistory(
             NetworkTemplate template, int uid, int set, int tag, int fields) {
+        return getHistory(template, uid, set, tag, fields, Long.MIN_VALUE, Long.MAX_VALUE);
+    }
+
+    /**
+     * Combine all {@link NetworkStatsHistory} in this collection which match
+     * the requested parameters.
+     */
+    public NetworkStatsHistory getHistory(
+            NetworkTemplate template, int uid, int set, int tag, int fields, long start, long end) {
         final NetworkStatsHistory combined = new NetworkStatsHistory(
                 mBucketDuration, estimateBuckets(), fields);
         for (Map.Entry<Key, NetworkStatsHistory> entry : mStats.entrySet()) {
@@ -128,7 +149,7 @@ public class NetworkStatsCollection implements FileRotator.Reader {
             final boolean setMatches = set == SET_ALL || key.set == set;
             if (key.uid == uid && setMatches && key.tag == tag
                     && templateMatches(template, key.ident)) {
-                combined.recordEntireHistory(entry.getValue());
+                combined.recordHistory(entry.getValue(), start, end);
             }
         }
         return combined;
@@ -145,6 +166,9 @@ public class NetworkStatsCollection implements FileRotator.Reader {
         final NetworkStats.Entry entry = new NetworkStats.Entry();
         NetworkStatsHistory.Entry historyEntry = null;
 
+        // shortcut when we know stats will be empty
+        if (start == end) return stats;
+
         for (Map.Entry<Key, NetworkStatsHistory> mapEntry : mStats.entrySet()) {
             final Key key = mapEntry.getKey();
             if (templateMatches(template, key.ident)) {
@@ -175,8 +199,9 @@ public class NetworkStatsCollection implements FileRotator.Reader {
      */
     public void recordData(NetworkIdentitySet ident, int uid, int set, int tag, long start,
             long end, NetworkStats.Entry entry) {
-        noteRecordedHistory(start, end, entry.rxBytes + entry.txBytes);
+        final NetworkStatsHistory history = findOrCreateHistory(ident, uid, set, tag);
-        findOrCreateHistory(ident, uid, set, tag).recordData(start, end, entry);
+        history.recordData(start, end, entry);
+        noteRecordedHistory(history.getStart(), history.getEnd(), entry.rxBytes + entry.txBytes);
     }
 
     /**

services/java/com/android/server/net/NetworkStatsService.java

@@ -44,6 +44,7 @@ import static android.provider.Settings.Secure.NETSTATS_DEV_PERSIST_BYTES;
 import static android.provider.Settings.Secure.NETSTATS_DEV_ROTATE_AGE;
 import static android.provider.Settings.Secure.NETSTATS_GLOBAL_ALERT_BYTES;
 import static android.provider.Settings.Secure.NETSTATS_POLL_INTERVAL;
+import static android.provider.Settings.Secure.NETSTATS_REPORT_XT_OVER_DEV;
 import static android.provider.Settings.Secure.NETSTATS_SAMPLE_ENABLED;
 import static android.provider.Settings.Secure.NETSTATS_TIME_CACHE_MAX_AGE;
 import static android.provider.Settings.Secure.NETSTATS_UID_BUCKET_DURATION;
@@ -177,6 +178,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         public long getPollInterval();
         public long getTimeCacheMaxAge();
         public boolean getSampleEnabled();
+        public boolean getReportXtOverDev();
 
         public static class Config {
             public final long bucketDuration;
@@ -221,6 +223,8 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
 
     /** Cached {@link #mDevRecorder} stats. */
     private NetworkStatsCollection mDevStatsCached;
+    /** Cached {@link #mXtRecorder} stats. */
+    private NetworkStatsCollection mXtStatsCached;
 
     /** Current counter sets for each UID. */
     private SparseIntArray mActiveUidCounterSet = new SparseIntArray();
@@ -295,6 +299,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             // read historical network stats from disk, since policy service
             // might need them right away.
             mDevStatsCached = mDevRecorder.getOrLoadCompleteLocked();
+            mXtStatsCached = mXtRecorder.getOrLoadCompleteLocked();
 
             // bootstrap initial stats to prevent double-counting later
             bootstrapStatsLocked();
@@ -371,6 +376,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         mUidTagRecorder = null;
 
         mDevStatsCached = null;
+        mXtStatsCached = null;
 
         mSystemReady = false;
     }
@@ -469,12 +475,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             @Override
             public NetworkStats getSummaryForNetwork(
                     NetworkTemplate template, long start, long end) {
-                return mDevStatsCached.getSummary(template, start, end);
+                return internalGetSummaryForNetwork(template, start, end);
             }
 
             @Override
             public NetworkStatsHistory getHistoryForNetwork(NetworkTemplate template, int fields) {
-                return mDevStatsCached.getHistory(template, UID_ALL, SET_ALL, TAG_NONE, fields);
+                return internalGetHistoryForNetwork(template, fields);
             }
 
             @Override
@@ -507,11 +513,56 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         };
     }
 
+    /**
+     * Return network summary, splicing between {@link #mDevStatsCached}
+     * and {@link #mXtStatsCached} when appropriate.
+     */
+    private NetworkStats internalGetSummaryForNetwork(
+            NetworkTemplate template, long start, long end) {
+        if (!mSettings.getReportXtOverDev()) {
+            // shortcut when XT reporting disabled
+            return mDevStatsCached.getSummary(template, start, end);
+        }
+
+        // splice stats between DEV and XT, switching over from DEV to XT at
+        // first atomic bucket.
+        final long firstAtomicBucket = mXtStatsCached.getFirstAtomicBucketMillis();
+        final NetworkStats dev = mDevStatsCached.getSummary(
+                template, Math.min(start, firstAtomicBucket), Math.min(end, firstAtomicBucket));
+        final NetworkStats xt = mXtStatsCached.getSummary(
+                template, Math.max(start, firstAtomicBucket), Math.max(end, firstAtomicBucket));
+
+        xt.combineAllValues(dev);
+        return xt;
+    }
+
+    /**
+     * Return network history, splicing between {@link #mDevStatsCached}
+     * and {@link #mXtStatsCached} when appropriate.
+     */
+    private NetworkStatsHistory internalGetHistoryForNetwork(NetworkTemplate template, int fields) {
+        if (!mSettings.getReportXtOverDev()) {
+            // shortcut when XT reporting disabled
+            return mDevStatsCached.getHistory(template, UID_ALL, SET_ALL, TAG_NONE, fields);
+        }
+
+        // splice stats between DEV and XT, switching over from DEV to XT at
+        // first atomic bucket.
+        final long firstAtomicBucket = mXtStatsCached.getFirstAtomicBucketMillis();
+        final NetworkStatsHistory dev = mDevStatsCached.getHistory(
+                template, UID_ALL, SET_ALL, TAG_NONE, fields, Long.MIN_VALUE, firstAtomicBucket);
+        final NetworkStatsHistory xt = mXtStatsCached.getHistory(
+                template, UID_ALL, SET_ALL, TAG_NONE, fields, firstAtomicBucket, Long.MAX_VALUE);
+
+        xt.recordEntireHistory(dev);
+        return xt;
+    }
+
     @Override
     public long getNetworkTotalBytes(NetworkTemplate template, long start, long end) {
         mContext.enforceCallingOrSelfPermission(READ_NETWORK_USAGE_HISTORY, TAG);
         assertBandwidthControlEnabled();
-        return mDevStatsCached.getSummary(template, start, end).getTotalBytes();
+        return internalGetSummaryForNetwork(template, start, end).getTotalBytes();
     }
 
     @Override
@@ -1190,6 +1241,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             return getSecureBoolean(NETSTATS_SAMPLE_ENABLED, true);
         }
         @Override
+        public boolean getReportXtOverDev() {
+            return getSecureBoolean(NETSTATS_REPORT_XT_OVER_DEV, true);
+        }
+        @Override
         public Config getDevConfig() {
             return new Config(getSecureLong(NETSTATS_DEV_BUCKET_DURATION, HOUR_IN_MILLIS),
                     getSecureLong(NETSTATS_DEV_ROTATE_AGE, 15 * DAY_IN_MILLIS),