Collect historical network stats.

Periodically records delta network traffic into historical buckets to
support other services, such NetworkPolicyManager and Settings UI.

Introduces NetworkStatsHistory structure which contains sparse, uniform
buckets of data usage defined by timestamps.  Service periodically
polls NetworkStats and records changes into buckets.  It only persists
to disk when substantial changes have occured.  Current parameters
create 4 buckets each day, and persist for 90 days, resulting in about
8kB of data per network.

Only records stats for "well known" network interfaces that have been
claimed by Telephony or Wi-Fi subsystems.  Historical stats are also
keyed off identity (such as IMSI) to support SIM swapping.

Change-Id: Ia27d1289556a2bf9545fbc4f3b789425a01be53a

core/java/android/net/INetworkStatsService.aidl

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net;
+
+import android.net.NetworkStatsHistory;
+
+/** {@hide} */
+interface INetworkStatsService {
+
+    NetworkStatsHistory[] getNetworkStatsSummary(int networkType);
+    NetworkStatsHistory getNetworkStatsUid(int uid);
+
+}

core/java/android/net/NetworkStats.java

@@ -22,19 +22,22 @@ import android.os.SystemClock;
 
 import java.io.CharArrayWriter;
 import java.io.PrintWriter;
+import java.util.HashSet;
 
 /**
- * Collection of network statistics. Can contain summary details across all
- * interfaces, or details with per-UID granularity. Designed to parcel quickly
- * across process boundaries.
+ * Collection of active network statistics. Can contain summary details across
+ * all interfaces, or details with per-UID granularity. Internally stores data
+ * as a large table, closely matching {@code /proc/} data format. This structure
+ * optimizes for rapid in-memory comparison, but consider using
+ * {@link NetworkStatsHistory} when persisting.
  *
  * @hide
  */
 public class NetworkStats implements Parcelable {
-    /** {@link #iface} value when entry is summarized over all interfaces. */
+    /** {@link #iface} value when interface details unavailable. */
     public static final String IFACE_ALL = null;
-    /** {@link #uid} value when entry is summarized over all UIDs. */
-    public static final int UID_ALL = 0;
+    /** {@link #uid} value when UID details unavailable. */
+    public static final int UID_ALL = -1;
 
     // NOTE: data should only be accounted for once in this structure; if data
     // is broken out, the summarized version should not be included.
@@ -49,7 +52,7 @@ public class NetworkStats implements Parcelable {
     public final long[] rx;
     public final long[] tx;
 
-    // TODO: add fg/bg stats and tag granularity
+    // TODO: add fg/bg stats once reported by kernel
 
     private NetworkStats(long elapsedRealtime, String[] iface, int[] uid, long[] rx, long[] tx) {
         this.elapsedRealtime = elapsedRealtime;
@@ -119,16 +122,36 @@ public class NetworkStats implements Parcelable {
         return -1;
     }
 
+    /**
+     * Return list of unique interfaces known by this data structure.
+     */
+    public String[] getKnownIfaces() {
+        final HashSet<String> ifaces = new HashSet<String>();
+        for (String iface : this.iface) {
+            if (iface != IFACE_ALL) {
+                ifaces.add(iface);
+            }
+        }
+        return ifaces.toArray(new String[ifaces.size()]);
+    }
+
     /**
      * Subtract the given {@link NetworkStats}, effectively leaving the delta
      * between two snapshots in time. Assumes that statistics rows collect over
      * time, and that none of them have disappeared.
+     *
+     * @param enforceMonotonic Validate that incoming value is strictly
+     *            monotonic compared to this object.
      */
-    public NetworkStats subtract(NetworkStats value) {
-        // result will have our rows, but no meaningful timestamp
-        final int length = length();
-        final NetworkStats.Builder result = new NetworkStats.Builder(-1, length);
+    public NetworkStats subtract(NetworkStats value, boolean enforceMonotonic) {
+        final long deltaRealtime = this.elapsedRealtime - value.elapsedRealtime;
+        if (enforceMonotonic && deltaRealtime < 0) {
+            throw new IllegalArgumentException("found non-monotonic realtime");
+        }
 
+        // result will have our rows, and elapsed time between snapshots
+        final int length = length();
+        final NetworkStats.Builder result = new NetworkStats.Builder(deltaRealtime, length);
         for (int i = 0; i < length; i++) {
             final String iface = this.iface[i];
             final int uid = this.uid[i];
@@ -142,6 +165,9 @@ public class NetworkStats implements Parcelable {
                 // existing row, subtract remote value
                 final long rx = this.rx[i] - value.rx[j];
                 final long tx = this.tx[i] - value.tx[j];
+                if (enforceMonotonic && (rx < 0 || tx < 0)) {
+                    throw new IllegalArgumentException("found non-monotonic values");
+                }
                 result.addEntry(iface, uid, rx, tx);
             }
         }

core/java/android/net/NetworkStatsHistory.aidl

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) 2011, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net;
+
+parcelable NetworkStatsHistory;

core/java/android/net/NetworkStatsHistory.java

@@ -0,0 +1,285 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+
+import java.io.CharArrayWriter;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+
+/**
+ * Collection of historical network statistics, recorded into equally-sized
+ * "buckets" in time. Internally it stores data in {@code long} series for more
+ * efficient persistence.
+ * <p>
+ * Each bucket is defined by a {@link #bucketStart} timestamp, and lasts for
+ * {@link #bucketDuration}. Internally assumes that {@link #bucketStart} is
+ * sorted at all times.
+ *
+ * @hide
+ */
+public class NetworkStatsHistory implements Parcelable {
+    private static final int VERSION = 1;
+
+    /** {@link #uid} value when UID details unavailable. */
+    public static final int UID_ALL = -1;
+
+    // TODO: teach about zigzag encoding to use less disk space
+    // TODO: teach how to convert between bucket sizes
+
+    public final int networkType;
+    public final String identity;
+    public final int uid;
+    public final long bucketDuration;
+
+    int bucketCount;
+    long[] bucketStart;
+    long[] rx;
+    long[] tx;
+
+    public NetworkStatsHistory(int networkType, String identity, int uid, long bucketDuration) {
+        this.networkType = networkType;
+        this.identity = identity;
+        this.uid = uid;
+        this.bucketDuration = bucketDuration;
+        bucketStart = new long[0];
+        rx = new long[0];
+        tx = new long[0];
+        bucketCount = bucketStart.length;
+    }
+
+    public NetworkStatsHistory(Parcel in) {
+        networkType = in.readInt();
+        identity = in.readString();
+        uid = in.readInt();
+        bucketDuration = in.readLong();
+        bucketStart = readLongArray(in);
+        rx = in.createLongArray();
+        tx = in.createLongArray();
+        bucketCount = bucketStart.length;
+    }
+
+    /** {@inheritDoc} */
+    public void writeToParcel(Parcel out, int flags) {
+        out.writeInt(networkType);
+        out.writeString(identity);
+        out.writeInt(uid);
+        out.writeLong(bucketDuration);
+        writeLongArray(out, bucketStart, bucketCount);
+        writeLongArray(out, rx, bucketCount);
+        writeLongArray(out, tx, bucketCount);
+    }
+
+    public NetworkStatsHistory(DataInputStream in) throws IOException {
+        final int version = in.readInt();
+        networkType = in.readInt();
+        identity = in.readUTF();
+        uid = in.readInt();
+        bucketDuration = in.readLong();
+        bucketStart = readLongArray(in);
+        rx = readLongArray(in);
+        tx = readLongArray(in);
+        bucketCount = bucketStart.length;
+    }
+
+    public void writeToStream(DataOutputStream out) throws IOException {
+        out.writeInt(VERSION);
+        out.writeInt(networkType);
+        out.writeUTF(identity);
+        out.writeInt(uid);
+        out.writeLong(bucketDuration);
+        writeLongArray(out, bucketStart, bucketCount);
+        writeLongArray(out, rx, bucketCount);
+        writeLongArray(out, tx, bucketCount);
+    }
+
+    /** {@inheritDoc} */
+    public int describeContents() {
+        return 0;
+    }
+
+    /**
+     * Record that data traffic occurred in the given time range. Will
+     * distribute across internal buckets, creating new buckets as needed.
+     */
+    public void recordData(long start, long end, long rx, long tx) {
+        // create any buckets needed by this range
+        ensureBuckets(start, end);
+
+        // distribute data usage into buckets
+        final long duration = end - start;
+        for (int i = bucketCount - 1; i >= 0; i--) {
+            final long curStart = bucketStart[i];
+            final long curEnd = curStart + bucketDuration;
+
+            // bucket is older than record; we're finished
+            if (curEnd < start) break;
+            // bucket is newer than record; keep looking
+            if (curStart > end) continue;
+
+            final long overlap = Math.min(curEnd, end) - Math.max(curStart, start);
+            if (overlap > 0) {
+                this.rx[i] += rx * overlap / duration;
+                this.tx[i] += tx * overlap / duration;
+            }
+        }
+    }
+
+    /**
+     * Ensure that buckets exist for given time range, creating as needed.
+     */
+    private void ensureBuckets(long start, long end) {
+        // normalize incoming range to bucket boundaries
+        start -= start % bucketDuration;
+        end += (bucketDuration - (end % bucketDuration)) % bucketDuration;
+
+        for (long now = start; now < end; now += bucketDuration) {
+            // try finding existing bucket
+            final int index = Arrays.binarySearch(bucketStart, 0, bucketCount, now);
+            if (index < 0) {
+                // bucket missing, create and insert
+                insertBucket(~index, now);
+            }
+        }
+    }
+
+    /**
+     * Insert new bucket at requested index and starting time.
+     */
+    private void insertBucket(int index, long start) {
+        // create more buckets when needed
+        if (bucketCount + 1 > bucketStart.length) {
+            final int newLength = bucketStart.length + 10;
+            bucketStart = Arrays.copyOf(bucketStart, newLength);
+            rx = Arrays.copyOf(rx, newLength);
+            tx = Arrays.copyOf(tx, newLength);
+        }
+
+        // create gap when inserting bucket in middle
+        if (index < bucketCount) {
+            final int dstPos = index + 1;
+            final int length = bucketCount - index;
+
+            System.arraycopy(bucketStart, index, bucketStart, dstPos, length);
+            System.arraycopy(rx, index, rx, dstPos, length);
+            System.arraycopy(tx, index, tx, dstPos, length);
+        }
+
+        bucketStart[index] = start;
+        rx[index] = 0;
+        tx[index] = 0;
+        bucketCount++;
+    }
+
+    /**
+     * Remove buckets older than requested cutoff.
+     */
+    public void removeBucketsBefore(long cutoff) {
+        int i;
+        for (i = 0; i < bucketCount; i++) {
+            final long curStart = bucketStart[i];
+            final long curEnd = curStart + bucketDuration;
+
+            // cutoff happens before or during this bucket; everything before
+            // this bucket should be removed.
+            if (curEnd > cutoff) break;
+        }
+
+        if (i > 0) {
+            final int length = bucketStart.length;
+            bucketStart = Arrays.copyOfRange(bucketStart, i, length);
+            rx = Arrays.copyOfRange(rx, i, length);
+            tx = Arrays.copyOfRange(tx, i, length);
+            bucketCount -= i;
+        }
+    }
+
+    public void dump(String prefix, PrintWriter pw) {
+        // TODO: consider stripping identity when dumping
+        pw.print(prefix);
+        pw.print("NetworkStatsHistory: networkType="); pw.print(networkType);
+        pw.print(" identity="); pw.print(identity);
+        pw.print(" uid="); pw.println(uid);
+        for (int i = 0; i < bucketCount; i++) {
+            pw.print(prefix);
+            pw.print("  timestamp="); pw.print(bucketStart[i]);
+            pw.print(" rx="); pw.print(rx[i]);
+            pw.print(" tx="); pw.println(tx[i]);
+        }
+    }
+
+    @Override
+    public String toString() {
+        final CharArrayWriter writer = new CharArrayWriter();
+        dump("", new PrintWriter(writer));
+        return writer.toString();
+    }
+
+    public static final Creator<NetworkStatsHistory> CREATOR = new Creator<NetworkStatsHistory>() {
+        public NetworkStatsHistory createFromParcel(Parcel in) {
+            return new NetworkStatsHistory(in);
+        }
+
+        public NetworkStatsHistory[] newArray(int size) {
+            return new NetworkStatsHistory[size];
+        }
+    };
+
+    private static long[] readLongArray(DataInputStream in) throws IOException {
+        final int size = in.readInt();
+        final long[] values = new long[size];
+        for (int i = 0; i < values.length; i++) {
+            values[i] = in.readLong();
+        }
+        return values;
+    }
+
+    private static void writeLongArray(DataOutputStream out, long[] values, int size) throws IOException {
+        if (size > values.length) {
+            throw new IllegalArgumentException("size larger than length");
+        }
+        out.writeInt(size);
+        for (int i = 0; i < size; i++) {
+            out.writeLong(values[i]);
+        }
+    }
+
+    private static long[] readLongArray(Parcel in) {
+        final int size = in.readInt();
+        final long[] values = new long[size];
+        for (int i = 0; i < values.length; i++) {
+            values[i] = in.readLong();
+        }
+        return values;
+    }
+
+    private static void writeLongArray(Parcel out, long[] values, int size) {
+        if (size > values.length) {
+            throw new IllegalArgumentException("size larger than length");
+        }
+        out.writeInt(size);
+        for (int i = 0; i < size; i++) {
+            out.writeLong(values[i]);
+        }
+    }
+
+}

core/java/android/net/TrafficStats.java

@@ -141,7 +141,8 @@ public class TrafficStats {
 
             // subtract starting values and return delta
             final NetworkStats profilingStop = getNetworkStatsForUid(context);
-            final NetworkStats profilingDelta = profilingStop.subtract(sActiveProfilingStart);
+            final NetworkStats profilingDelta = profilingStop.subtract(
+                    sActiveProfilingStart, false);
             sActiveProfilingStart = null;
             return profilingDelta;
         }