Begin collecting xtables iface counters.

Add method to parse new iface_stat_fmt proc stats, or return null
when kernel support is unavailable. Add test and remove older, unused
parsing code. Create new "xt" recorder to persist the new xtables
counters when available.

Add SSID support to NetworkIdentity to fix policy tests.

Bug: 6422414
Change-Id: I77f70e9acb79a559ab626f3af5c4f3599801ed43

core/java/android/net/NetworkIdentity.java

@@ -158,10 +158,15 @@ public class NetworkIdentity {
             }
 
         } else if (type == TYPE_WIFI) {
-            final WifiManager wifi = (WifiManager) context.getSystemService(Context.WIFI_SERVICE);
+            if (state.networkId != null) {
+                networkId = state.networkId;
+            } else {
+                final WifiManager wifi = (WifiManager) context.getSystemService(
+                        Context.WIFI_SERVICE);
                 final WifiInfo info = wifi.getConnectionInfo();
                 networkId = info != null ? info.getSSID() : null;
             }
+        }
 
         return new NetworkIdentity(type, subType, subscriberId, networkId, roaming);
     }

core/java/com/android/internal/net/NetworkStatsFactory.java

@@ -24,20 +24,12 @@ import static com.android.server.NetworkManagementSocketTagger.kernelToTag;
 import android.net.NetworkStats;
 import android.os.StrictMode;
 import android.os.SystemClock;
-import android.util.Slog;
 
 import com.android.internal.util.ProcFileReader;
-import com.google.android.collect.Lists;
-import com.google.android.collect.Sets;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileReader;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.StringTokenizer;
 
 import libcore.io.IoUtils;
 
@@ -50,14 +42,10 @@ public class NetworkStatsFactory {
 
     // TODO: consider moving parsing to native code
 
-    /** Path to {@code /proc/net/dev}. */
-    @Deprecated
-    private final File mStatsIface;
-    /** Path to {@code /proc/net/xt_qtaguid/iface_stat}. */
-    @Deprecated
-    private final File mStatsXtIface;
     /** Path to {@code /proc/net/xt_qtaguid/iface_stat_all}. */
     private final File mStatsXtIfaceAll;
+    /** Path to {@code /proc/net/xt_qtaguid/iface_stat_fmt}. */
+    private final File mStatsXtIfaceFmt;
     /** Path to {@code /proc/net/xt_qtaguid/stats}. */
     private final File mStatsXtUid;
 
@@ -67,28 +55,20 @@ public class NetworkStatsFactory {
 
     // @VisibleForTesting
     public NetworkStatsFactory(File procRoot) {
-        mStatsIface = new File(procRoot, "net/dev");
-        mStatsXtUid = new File(procRoot, "net/xt_qtaguid/stats");
-        mStatsXtIface = new File(procRoot, "net/xt_qtaguid/iface_stat");
         mStatsXtIfaceAll = new File(procRoot, "net/xt_qtaguid/iface_stat_all");
+        mStatsXtIfaceFmt = new File(procRoot, "net/xt_qtaguid/iface_stat_fmt");
+        mStatsXtUid = new File(procRoot, "net/xt_qtaguid/stats");
     }
 
     /**
-     * Parse and return interface-level summary {@link NetworkStats}. Values
+     * Parse and return interface-level summary {@link NetworkStats} measured
-     * monotonically increase since device boot, and may include details about
+     * using {@code /proc/net/dev} style hooks, which may include non IP layer
-     * inactive interfaces.
+     * traffic. Values monotonically increase since device boot, and may include
+     * details about inactive interfaces.
      *
      * @throws IllegalStateException when problem parsing stats.
      */
-    public NetworkStats readNetworkStatsSummary() throws IllegalStateException {
+    public NetworkStats readNetworkStatsSummaryDev() throws IllegalStateException {
-        if (mStatsXtIfaceAll.exists()) {
-            return readNetworkStatsSummarySingleFile();
-        } else {
-            return readNetworkStatsSummaryMultipleFiles();
-        }
-    }
-
-    private NetworkStats readNetworkStatsSummarySingleFile() {
         final StrictMode.ThreadPolicy savedPolicy = StrictMode.allowThreadDiskReads();
 
         final NetworkStats stats = new NetworkStats(SystemClock.elapsedRealtime(), 6);
@@ -137,79 +117,40 @@ public class NetworkStatsFactory {
     }
 
     /**
-     * @deprecated remove once {@code iface_stat_all} is merged to all kernels.
+     * Parse and return interface-level summary {@link NetworkStats}. Designed
+     * to return only IP layer traffic. Values monotonically increase since
+     * device boot, and may include details about inactive interfaces.
+     *
+     * @throws IllegalStateException when problem parsing stats.
      */
-    @Deprecated
+    public NetworkStats readNetworkStatsSummaryXt() throws IllegalStateException {
-    private NetworkStats readNetworkStatsSummaryMultipleFiles() {
         final StrictMode.ThreadPolicy savedPolicy = StrictMode.allowThreadDiskReads();
 
+        // return null when kernel doesn't support
+        if (!mStatsXtIfaceFmt.exists()) return null;
+
         final NetworkStats stats = new NetworkStats(SystemClock.elapsedRealtime(), 6);
         final NetworkStats.Entry entry = new NetworkStats.Entry();
 
-        final HashSet<String> knownIfaces = Sets.newHashSet();
+        ProcFileReader reader = null;
-        final HashSet<String> activeIfaces = Sets.newHashSet();
+        try {
+            // open and consume header line
+            reader = new ProcFileReader(new FileInputStream(mStatsXtIfaceFmt));
+            reader.finishLine();
 
-        // collect any historical stats and active state
+            while (reader.hasMoreData()) {
-        for (String iface : fileListWithoutNull(mStatsXtIface)) {
+                entry.iface = reader.nextString();
-            final File ifacePath = new File(mStatsXtIface, iface);
-
-            final long active = readSingleLongFromFile(new File(ifacePath, "active"));
-            if (active == 1) {
-                knownIfaces.add(iface);
-                activeIfaces.add(iface);
-            } else if (active == 0) {
-                knownIfaces.add(iface);
-            } else {
-                continue;
-            }
-
-            entry.iface = iface;
                 entry.uid = UID_ALL;
                 entry.set = SET_ALL;
                 entry.tag = TAG_NONE;
-            entry.rxBytes = readSingleLongFromFile(new File(ifacePath, "rx_bytes"));
+
-            entry.rxPackets = readSingleLongFromFile(new File(ifacePath, "rx_packets"));
+                entry.rxBytes = reader.nextLong();
-            entry.txBytes = readSingleLongFromFile(new File(ifacePath, "tx_bytes"));
+                entry.rxPackets = reader.nextLong();
-            entry.txPackets = readSingleLongFromFile(new File(ifacePath, "tx_packets"));
+                entry.txBytes = reader.nextLong();
+                entry.txPackets = reader.nextLong();
 
                 stats.addValues(entry);
-        }
+                reader.finishLine();
-
-        final ArrayList<String> values = Lists.newArrayList();
-
-        BufferedReader reader = null;
-        try {
-            reader = new BufferedReader(new FileReader(mStatsIface));
-
-            // skip first two header lines
-            reader.readLine();
-            reader.readLine();
-
-            // parse remaining lines
-            String line;
-            while ((line = reader.readLine()) != null) {
-                splitLine(line, values);
-
-                try {
-                    entry.iface = values.get(0);
-                    entry.uid = UID_ALL;
-                    entry.set = SET_ALL;
-                    entry.tag = TAG_NONE;
-                    entry.rxBytes = Long.parseLong(values.get(1));
-                    entry.rxPackets = Long.parseLong(values.get(2));
-                    entry.txBytes = Long.parseLong(values.get(9));
-                    entry.txPackets = Long.parseLong(values.get(10));
-
-                    if (activeIfaces.contains(entry.iface)) {
-                        // combine stats when iface is active
-                        stats.combineValues(entry);
-                    } else if (!knownIfaces.contains(entry.iface)) {
-                        // add stats when iface is unknown
-                        stats.addValues(entry);
-                    }
-                } catch (NumberFormatException e) {
-                    Slog.w(TAG, "problem parsing stats row '" + line + "': " + e);
-                }
             }
         } catch (NullPointerException e) {
             throw new IllegalStateException("problem parsing stats: " + e);
@@ -221,7 +162,6 @@ public class NetworkStatsFactory {
             IoUtils.closeQuietly(reader);
             StrictMode.setThreadPolicy(savedPolicy);
         }
-
         return stats;
     }
 
@@ -286,41 +226,4 @@ public class NetworkStatsFactory {
 
         return stats;
     }
-
-    /**
-     * Split given line into {@link ArrayList}.
-     */
-    @Deprecated
-    private static void splitLine(String line, ArrayList<String> outSplit) {
-        outSplit.clear();
-
-        final StringTokenizer t = new StringTokenizer(line, " \t\n\r\f:");
-        while (t.hasMoreTokens()) {
-            outSplit.add(t.nextToken());
-        }
-    }
-
-    /**
-     * Utility method to read a single plain-text {@link Long} from the given
-     * {@link File}, usually from a {@code /proc/} filesystem.
-     */
-    private static long readSingleLongFromFile(File file) {
-        try {
-            final byte[] buffer = IoUtils.readFileAsByteArray(file.toString());
-            return Long.parseLong(new String(buffer).trim());
-        } catch (NumberFormatException e) {
-            return -1;
-        } catch (IOException e) {
-            return -1;
-        }
-    }
-
-    /**
-     * Wrapper for {@link File#list()} that returns empty array instead of
-     * {@code null}.
-     */
-    private static String[] fileListWithoutNull(File file) {
-        final String[] list = file.list();
-        return list != null ? list : new String[0];
-    }
 }

services/java/com/android/server/net/NetworkStatsRecorder.java

@@ -128,6 +128,9 @@ public class NetworkStatsRecorder {
             Map<String, NetworkIdentitySet> ifaceIdent, long currentTimeMillis) {
         final HashSet<String> unknownIfaces = Sets.newHashSet();
 
+        // skip recording when snapshot missing
+        if (snapshot == null) return;
+
         // assume first snapshot is bootstrap and don't record
         if (mLastSnapshot == null) {
             mLastSnapshot = snapshot;

services/java/com/android/server/net/NetworkStatsService.java

@@ -159,6 +159,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     private PendingIntent mPollIntent;
 
     private static final String PREFIX_DEV = "dev";
+    private static final String PREFIX_XT = "xt";
     private static final String PREFIX_UID = "uid";
     private static final String PREFIX_UID_TAG = "uid_tag";
 
@@ -187,6 +188,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         }
 
         public Config getDevConfig();
+        public Config getXtConfig();
         public Config getUidConfig();
         public Config getUidTagConfig();
     }
@@ -204,6 +206,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             new DropBoxNonMonotonicObserver();
 
     private NetworkStatsRecorder mDevRecorder;
+    private NetworkStatsRecorder mXtRecorder;
     private NetworkStatsRecorder mUidRecorder;
     private NetworkStatsRecorder mUidTagRecorder;
 
@@ -268,6 +271,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
 
         // create data recorders along with historical rotators
         mDevRecorder = buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false);
+        mXtRecorder = buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false);
         mUidRecorder = buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false);
         mUidTagRecorder = buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true);
 
@@ -343,10 +347,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
 
         // persist any pending stats
         mDevRecorder.forcePersistLocked(currentTime);
+        mXtRecorder.forcePersistLocked(currentTime);
         mUidRecorder.forcePersistLocked(currentTime);
         mUidTagRecorder.forcePersistLocked(currentTime);
 
         mDevRecorder = null;
+        mXtRecorder = null;
         mUidRecorder = null;
         mUidTagRecorder = null;
 
@@ -772,9 +778,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             // snapshot and record current counters; read UID stats first to
             // avoid overcounting dev stats.
             final NetworkStats uidSnapshot = getNetworkStatsUidDetail();
-            final NetworkStats devSnapshot = getNetworkStatsSummary();
+            final NetworkStats xtSnapshot = mNetworkManager.getNetworkStatsSummaryXt();
+            final NetworkStats devSnapshot = mNetworkManager.getNetworkStatsSummaryDev();
 
             mDevRecorder.recordSnapshotLocked(devSnapshot, mActiveIfaces, currentTime);
+            mXtRecorder.recordSnapshotLocked(xtSnapshot, mActiveIfaces, currentTime);
             mUidRecorder.recordSnapshotLocked(uidSnapshot, mActiveIfaces, currentTime);
             mUidTagRecorder.recordSnapshotLocked(uidSnapshot, mActiveIfaces, currentTime);
 
@@ -824,9 +832,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             // snapshot and record current counters; read UID stats first to
             // avoid overcounting dev stats.
             final NetworkStats uidSnapshot = getNetworkStatsUidDetail();
-            final NetworkStats devSnapshot = getNetworkStatsSummary();
+            final NetworkStats xtSnapshot = mNetworkManager.getNetworkStatsSummaryXt();
+            final NetworkStats devSnapshot = mNetworkManager.getNetworkStatsSummaryDev();
 
             mDevRecorder.recordSnapshotLocked(devSnapshot, mActiveIfaces, currentTime);
+            mXtRecorder.recordSnapshotLocked(xtSnapshot, mActiveIfaces, currentTime);
             mUidRecorder.recordSnapshotLocked(uidSnapshot, mActiveIfaces, currentTime);
             mUidTagRecorder.recordSnapshotLocked(uidSnapshot, mActiveIfaces, currentTime);
 
@@ -841,11 +851,13 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         // persist any pending data depending on requested flags
         if (persistForce) {
             mDevRecorder.forcePersistLocked(currentTime);
+            mXtRecorder.forcePersistLocked(currentTime);
             mUidRecorder.forcePersistLocked(currentTime);
             mUidTagRecorder.forcePersistLocked(currentTime);
         } else {
             if (persistNetwork) {
                 mDevRecorder.maybePersistLocked(currentTime);
+                mXtRecorder.maybePersistLocked(currentTime);
             }
             if (persistUid) {
                 mUidRecorder.maybePersistLocked(currentTime);
@@ -884,7 +896,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         // collect mobile sample
         template = buildTemplateMobileWildcard();
         devTotal = mDevRecorder.getTotalSinceBootLocked(template);
-        xtTotal = new NetworkStats.Entry();
+        xtTotal = mXtRecorder.getTotalSinceBootLocked(template);
         uidTotal = mUidRecorder.getTotalSinceBootLocked(template);
 
         EventLogTags.writeNetstatsMobileSample(
@@ -896,7 +908,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         // collect wifi sample
         template = buildTemplateWifiWildcard();
         devTotal = mDevRecorder.getTotalSinceBootLocked(template);
-        xtTotal = new NetworkStats.Entry();
+        xtTotal = mXtRecorder.getTotalSinceBootLocked(template);
         uidTotal = mUidRecorder.getTotalSinceBootLocked(template);
 
         EventLogTags.writeNetstatsWifiSample(
@@ -970,6 +982,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
             mDevRecorder.dumpLocked(pw, fullHistory);
             pw.decreaseIndent();
 
+            pw.println("Xt stats:");
+            pw.increaseIndent();
+            mXtRecorder.dumpLocked(pw, fullHistory);
+            pw.decreaseIndent();
+
             if (includeUid) {
                 pw.println("UID stats:");
                 pw.increaseIndent();
@@ -986,10 +1003,6 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
         }
     }
 
-    private NetworkStats getNetworkStatsSummary() throws RemoteException {
-        return mNetworkManager.getNetworkStatsSummary();
-    }
-
     /**
      * Return snapshot of current UID statistics, including any
      * {@link TrafficStats#UID_TETHERING} and {@link #mUidOperations} values.
@@ -1125,6 +1138,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
                     getSecureLong(NETSTATS_DEV_DELETE_AGE, 90 * DAY_IN_MILLIS));
         }
 
+        @Override
+        public Config getXtConfig() {
+            return getDevConfig();
+        }
+
         @Override
         public Config getUidConfig() {
             return new Config(getSecureLong(NETSTATS_UID_BUCKET_DURATION, 2 * HOUR_IN_MILLIS),