Persist network stats using AtomicFile.

Implements read/write of network stats using AtomicFile, along with
magic number and versioning.  Stores in "/data/system/netstats.bin"
for now.  Tests to verify that stats are persisted across a simulated
reboot, and to verify that TEMPLATE_WIFI is working.

Fixed bug where kernel counters rolling backwards would cause negative
stats to be recorded; now we clamp deltas at 0.

Change-Id: I53bce26fc8fd3f4ab1e34ce135d302edfa34db34

core/java/android/net/INetworkStatsService.aidl

@@ -28,6 +28,6 @@ interface INetworkStatsService {
     NetworkStatsHistory getHistoryForUid(int uid, int networkTemplate);
 
     /** Return usage summary per UID for traffic that matches template. */
-    NetworkStats getSummaryPerUid(long start, long end, int networkTemplate);
+    NetworkStats getSummaryForAllUid(long start, long end, int networkTemplate);
 
 }

core/java/android/net/NetworkStats.java

@@ -153,6 +153,30 @@ public class NetworkStats implements Parcelable {
         return result;
     }
 
+    /**
+     * Subtract the given {@link NetworkStats}, effectively leaving the delta
+     * between two snapshots in time. Assumes that statistics rows collect over
+     * time, and that none of them have disappeared.
+     *
+     * @throws IllegalArgumentException when given {@link NetworkStats} is
+     *             non-monotonic.
+     */
+    public NetworkStats subtract(NetworkStats value) {
+        return subtract(value, true, false);
+    }
+
+    /**
+     * Subtract the given {@link NetworkStats}, effectively leaving the delta
+     * between two snapshots in time. Assumes that statistics rows collect over
+     * time, and that none of them have disappeared.
+     * <p>
+     * Instead of throwing when counters are non-monotonic, this variant clamps
+     * results to never be negative.
+     */
+    public NetworkStats subtractClamped(NetworkStats value) {
+        return subtract(value, false, true);
+    }
+
     /**
      * Subtract the given {@link NetworkStats}, effectively leaving the delta
      * between two snapshots in time. Assumes that statistics rows collect over
@@ -160,8 +184,11 @@ public class NetworkStats implements Parcelable {
      *
      * @param enforceMonotonic Validate that incoming value is strictly
      *            monotonic compared to this object.
+     * @param clampNegative Instead of throwing like {@code enforceMonotonic},
+     *            clamp resulting counters at 0 to prevent negative values.
      */
-    public NetworkStats subtract(NetworkStats value, boolean enforceMonotonic) {
+    private NetworkStats subtract(
+            NetworkStats value, boolean enforceMonotonic, boolean clampNegative) {
         final long deltaRealtime = this.elapsedRealtime - value.elapsedRealtime;
         if (enforceMonotonic && deltaRealtime < 0) {
             throw new IllegalArgumentException("found non-monotonic realtime");
@@ -181,11 +208,15 @@ public class NetworkStats implements Parcelable {
                 result.addEntry(iface, uid, this.rx[i], this.tx[i]);
             } else {
                 // existing row, subtract remote value
-                final long rx = this.rx[i] - value.rx[j];
+                long rx = this.rx[i] - value.rx[j];
-                final long tx = this.tx[i] - value.tx[j];
+                long tx = this.tx[i] - value.tx[j];
                 if (enforceMonotonic && (rx < 0 || tx < 0)) {
                     throw new IllegalArgumentException("found non-monotonic values");
                 }
+                if (clampNegative) {
+                    rx = Math.max(0, rx);
+                    tx = Math.max(0, tx);
+                }
                 result.addEntry(iface, uid, rx, tx);
             }
         }

core/java/android/net/NetworkStatsHistory.java

@@ -218,7 +218,7 @@ public class NetworkStatsHistory implements Parcelable {
      * Return interpolated data usage across the requested range. Interpolates
      * across buckets, so values may be rounded slightly.
      */
-    public void getTotalData(long start, long end, long[] outTotal) {
+    public long[] getTotalData(long start, long end, long[] outTotal) {
         long rx = 0;
         long tx = 0;
 
@@ -238,8 +238,12 @@ public class NetworkStatsHistory implements Parcelable {
             }
         }
 
+        if (outTotal == null || outTotal.length != 2) {
+            outTotal = new long[2];
+        }
         outTotal[0] = rx;
         outTotal[1] = tx;
+        return outTotal;
     }
 
     /**

core/java/android/net/TrafficStats.java

@@ -177,8 +177,8 @@ public class TrafficStats {
 
             // subtract starting values and return delta
             final NetworkStats profilingStop = getNetworkStatsForUid(context);
-            final NetworkStats profilingDelta = profilingStop.subtract(
+            final NetworkStats profilingDelta = profilingStop.subtractClamped(
-                    sActiveProfilingStart, false);
+                    sActiveProfilingStart);
             sActiveProfilingStart = null;
             return profilingDelta;
         }

services/java/com/android/server/net/NetworkStatsService.java

@@ -48,6 +48,7 @@ import android.net.NetworkInfo;
 import android.net.NetworkState;
 import android.net.NetworkStats;
 import android.net.NetworkStatsHistory;
+import android.os.Environment;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.INetworkManagementService;
@@ -60,15 +61,26 @@ import android.util.Slog;
 import android.util.SparseArray;
 import android.util.TrustedTime;
 
+import com.android.internal.os.AtomicFile;
 import com.google.android.collect.Lists;
 import com.google.android.collect.Maps;
 
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
 import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.net.ProtocolException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 
+import libcore.io.IoUtils;
+
 /**
  * Collect and persist detailed network statistics, and provide this data to
  * other system services.
@@ -77,6 +89,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     private static final String TAG = "NetworkStatsService";
     private static final boolean LOGD = true;
 
+    /** File header magic number: "ANET" */
+    private static final int FILE_MAGIC = 0x414E4554;
+    private static final int VERSION_CURRENT = 1;
+
     private final Context mContext;
     private final INetworkManagementService mNetworkManager;
     private final IAlarmManager mAlarmManager;
@@ -84,7 +100,8 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
 
     private IConnectivityManager mConnManager;
 
-    private static final String ACTION_NETWORK_STATS_POLL =
+    // @VisibleForTesting
+    public static final String ACTION_NETWORK_STATS_POLL =
             "com.android.server.action.NETWORK_STATS_POLL";
 
     private PendingIntent mPollIntent;
@@ -98,14 +115,15 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     private LongSecureSetting mPollInterval = new LongSecureSetting(
             NETSTATS_POLL_INTERVAL, 15 * MINUTE_IN_MILLIS);
     private LongSecureSetting mPersistThreshold = new LongSecureSetting(
-            NETSTATS_PERSIST_THRESHOLD, 64 * KB_IN_BYTES);
+            NETSTATS_PERSIST_THRESHOLD, 16 * KB_IN_BYTES);
 
+    // TODO: adjust these timings for production builds
     private LongSecureSetting mSummaryBucketDuration = new LongSecureSetting(
-            NETSTATS_SUMMARY_BUCKET_DURATION, 6 * HOUR_IN_MILLIS);
+            NETSTATS_SUMMARY_BUCKET_DURATION, 1 * HOUR_IN_MILLIS);
     private LongSecureSetting mSummaryMaxHistory = new LongSecureSetting(
             NETSTATS_SUMMARY_MAX_HISTORY, 90 * DAY_IN_MILLIS);
     private LongSecureSetting mDetailBucketDuration = new LongSecureSetting(
-            NETSTATS_DETAIL_BUCKET_DURATION, 6 * HOUR_IN_MILLIS);
+            NETSTATS_DETAIL_BUCKET_DURATION, 2 * HOUR_IN_MILLIS);
     private LongSecureSetting mDetailMaxHistory = new LongSecureSetting(
             NETSTATS_DETAIL_MAX_HISTORY, 90 * DAY_IN_MILLIS);
 
@@ -129,6 +147,8 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     private final HandlerThread mHandlerThread;
     private final Handler mHandler;
 
+    private final AtomicFile mSummaryFile;
+
     // TODO: collect detailed uid stats, storing tag-granularity data until next
     // dropbox, and uid summary for a specific bucket count.
 
@@ -137,11 +157,15 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     public NetworkStatsService(
             Context context, INetworkManagementService networkManager, IAlarmManager alarmManager) {
         // TODO: move to using cached NtpTrustedTime
-        this(context, networkManager, alarmManager, new NtpTrustedTime());
+        this(context, networkManager, alarmManager, new NtpTrustedTime(), getSystemDir());
+    }
+
+    private static File getSystemDir() {
+        return new File(Environment.getDataDirectory(), "system");
     }
 
     public NetworkStatsService(Context context, INetworkManagementService networkManager,
-            IAlarmManager alarmManager, TrustedTime time) {
+            IAlarmManager alarmManager, TrustedTime time, File systemDir) {
         mContext = checkNotNull(context, "missing Context");
         mNetworkManager = checkNotNull(networkManager, "missing INetworkManagementService");
         mAlarmManager = checkNotNull(alarmManager, "missing IAlarmManager");
@@ -150,11 +174,15 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         mHandlerThread = new HandlerThread(TAG);
         mHandlerThread.start();
         mHandler = new Handler(mHandlerThread.getLooper());
+
+        mSummaryFile = new AtomicFile(new File(systemDir, "netstats.bin"));
     }
 
     public void systemReady() {
-        // read historical stats from disk
+        synchronized (mStatsLock) {
-        readStatsLocked();
+            // read historical stats from disk
+            readStatsLocked();
+        }
 
         // watch other system services that claim interfaces
         final IntentFilter ifaceFilter = new IntentFilter();
@@ -180,6 +208,16 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         mConnManager = checkNotNull(connManager, "missing IConnectivityManager");
     }
 
+    private void shutdownLocked() {
+        mContext.unregisterReceiver(mIfaceReceiver);
+        mContext.unregisterReceiver(mPollReceiver);
+        mContext.unregisterReceiver(mShutdownReceiver);
+
+        writeStatsLocked();
+        mSummaryStats.clear();
+        mDetailStats.clear();
+    }
+
     /**
      * Clear any existing {@link #ACTION_NETWORK_STATS_POLL} alarms, and
      * reschedule based on current {@link #mPollInterval} value.
@@ -227,7 +265,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     }
 
     @Override
-    public NetworkStats getSummaryPerUid(long start, long end, int networkTemplate) {
+    public NetworkStats getSummaryForAllUid(long start, long end, int networkTemplate) {
         // TODO: create relaxed permission for reading stats
         mContext.enforceCallingOrSelfPermission(UPDATE_DEVICE_STATS, TAG);
 
@@ -281,7 +319,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         public void onReceive(Context context, Intent intent) {
             // verified SHUTDOWN permission above.
             synchronized (mStatsLock) {
-                writeStatsLocked();
+                shutdownLocked();
             }
         }
     };
@@ -440,13 +478,74 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
 
     private void readStatsLocked() {
         if (LOGD) Slog.v(TAG, "readStatsLocked()");
-        // TODO: read historical stats from disk using AtomicFile
+
+        // clear any existing stats and read from disk
+        mSummaryStats.clear();
+
+        FileInputStream fis = null;
+        try {
+            fis = mSummaryFile.openRead();
+            final DataInputStream in = new DataInputStream(fis);
+
+            // verify file magic header intact
+            final int magic = in.readInt();
+            if (magic != FILE_MAGIC) {
+                throw new ProtocolException("unexpected magic: " + magic);
+            }
+
+            final int version = in.readInt();
+            switch (version) {
+                case VERSION_CURRENT: {
+                    // file format is pairs of interfaces and stats:
+                    // summary := size *(InterfaceIdentity NetworkStatsHistory)
+
+                    final int size = in.readInt();
+                    for (int i = 0; i < size; i++) {
+                        final InterfaceIdentity ident = new InterfaceIdentity(in);
+                        final NetworkStatsHistory history = new NetworkStatsHistory(in);
+                        mSummaryStats.put(ident, history);
+                    }
+                    break;
+                }
+                default: {
+                    throw new ProtocolException("unexpected version: " + version);
+                }
+            }
+        } catch (FileNotFoundException e) {
+            // missing stats is okay, probably first boot
+        } catch (IOException e) {
+            Slog.e(TAG, "problem reading network stats", e);
+        } finally {
+            IoUtils.closeQuietly(fis);
+        }
     }
 
     private void writeStatsLocked() {
         if (LOGD) Slog.v(TAG, "writeStatsLocked()");
-        // TODO: persist historical stats to disk using AtomicFile
+
         // TODO: consider duplicating stats and releasing lock while writing
+
+        FileOutputStream fos = null;
+        try {
+            fos = mSummaryFile.startWrite();
+            final DataOutputStream out = new DataOutputStream(fos);
+
+            out.writeInt(FILE_MAGIC);
+            out.writeInt(VERSION_CURRENT);
+
+            out.writeInt(mSummaryStats.size());
+            for (InterfaceIdentity ident : mSummaryStats.keySet()) {
+                final NetworkStatsHistory history = mSummaryStats.get(ident);
+                ident.writeToStream(out);
+                history.writeToStream(out);
+            }
+
+            mSummaryFile.finishWrite(fos);
+        } catch (IOException e) {
+            if (fos != null) {
+                mSummaryFile.failWrite(fos);
+            }
+        }
     }
 
     @Override
@@ -466,6 +565,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
                 return;
             }
 
+            if (argSet.contains("poll")) {
+                performPollLocked();
+                pw.println("Forced poll");
+                return;
+            }
+
             pw.println("Active interfaces:");
             for (String iface : mActiveIface.keySet()) {
                 final InterfaceIdentity ident = mActiveIface.get(iface);
@@ -545,7 +650,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
      */
     private static NetworkStats computeStatsDelta(NetworkStats before, NetworkStats current) {
         if (before != null) {
-            return current.subtract(before, false);
+            return current.subtractClamped(before);
         } else {
             return current;
         }