From b0efeaf8551e2867d771bff408ec83bead901631 Mon Sep 17 00:00:00 2001 From: Lorenzo Colitti Date: Mon, 2 Apr 2018 16:48:31 +0900 Subject: [PATCH 1/2] Allow applications to query for foreground/background data usage. Currently the NetworkStatsManager APIs allow applications to query for their own data usage by UID and tag, but do not allow applications to query by foreground/background state. This is causing popular apps to resort to parsing xt_qtaguid stats files directly. Because this is no longer allowed for apps targeting P and above, provide replacement functionality. This API allows apps to query for data usage for a given state, but not to receive data usage broken down by state. This is consistent with how the current UID and tag APIs work. It is also not an undue burden on apps: there are currently only two states of interest (FOREGROUND and everything else), and even if we add states in the future, unmodified apps can still obtain total traffic using STATE_ALL. Bug: 72977484 Test: New CTS test added in other change in this topic. Change-Id: Ic8c9194569ffd599b49e4a8197c5c2ea0ec3f7f7 --- core/java/android/app/usage/NetworkStats.java | 35 ++++++++++------- .../app/usage/NetworkStatsManager.java | 38 ++++++++++++------- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/core/java/android/app/usage/NetworkStats.java b/core/java/android/app/usage/NetworkStats.java index e315e9115f..7252f028d7 100644 --- a/core/java/android/app/usage/NetworkStats.java +++ b/core/java/android/app/usage/NetworkStats.java @@ -67,6 +67,11 @@ public final class NetworkStats implements AutoCloseable { */ private int mTag = android.net.NetworkStats.TAG_NONE; + /** + * State in case it was not specified in the query. + */ + private int mState = Bucket.STATE_ALL; + /** * The session while the query requires it, null if all the stats have been collected or close() * has been called. @@ -267,6 +272,15 @@ public final class NetworkStats implements AutoCloseable { private long mTxBytes; private long mTxPackets; + private static int convertSet(@State int state) { + switch (state) { + case STATE_ALL: return android.net.NetworkStats.SET_ALL; + case STATE_DEFAULT: return android.net.NetworkStats.SET_DEFAULT; + case STATE_FOREGROUND: return android.net.NetworkStats.SET_FOREGROUND; + } + return 0; + } + private static @State int convertState(int networkStatsSet) { switch (networkStatsSet) { case android.net.NetworkStats.SET_ALL : return STATE_ALL; @@ -527,20 +541,13 @@ public final class NetworkStats implements AutoCloseable { /** * Collects history results for uid and resets history enumeration index. */ - void startHistoryEnumeration(int uid) { - startHistoryEnumeration(uid, android.net.NetworkStats.TAG_NONE); - } - - /** - * Collects history results for uid and resets history enumeration index. - */ - void startHistoryEnumeration(int uid, int tag) { + void startHistoryEnumeration(int uid, int tag, int state) { mHistory = null; try { mHistory = mSession.getHistoryIntervalForUid(mTemplate, uid, - android.net.NetworkStats.SET_ALL, tag, - NetworkStatsHistory.FIELD_ALL, mStartTimeStamp, mEndTimeStamp); - setSingleUidTag(uid, tag); + Bucket.convertSet(state), tag, NetworkStatsHistory.FIELD_ALL, + mStartTimeStamp, mEndTimeStamp); + setSingleUidTagState(uid, tag, state); } catch (RemoteException e) { Log.w(TAG, e); // Leaving mHistory null @@ -636,6 +643,7 @@ public final class NetworkStats implements AutoCloseable { fillBucketFromSummaryEntry(bucket); return bucket; } + /** * Getting the next item in a history enumeration. * @param bucketOut Next item will be set here. @@ -648,7 +656,7 @@ public final class NetworkStats implements AutoCloseable { mRecycledHistoryEntry); bucketOut.mUid = Bucket.convertUid(getUid()); bucketOut.mTag = Bucket.convertTag(mTag); - bucketOut.mState = Bucket.STATE_ALL; + bucketOut.mState = mState; bucketOut.mDefaultNetwork = Bucket.DEFAULT_NETWORK_ALL; bucketOut.mMetered = Bucket.METERED_ALL; bucketOut.mRoaming = Bucket.ROAMING_ALL; @@ -691,9 +699,10 @@ public final class NetworkStats implements AutoCloseable { return mUidOrUidIndex; } - private void setSingleUidTag(int uid, int tag) { + private void setSingleUidTagState(int uid, int tag, int state) { mUidOrUidIndex = uid; mTag = tag; + mState = state; } private void stepUid() { diff --git a/core/java/android/app/usage/NetworkStatsManager.java b/core/java/android/app/usage/NetworkStatsManager.java index 2357637b72..b2fe958691 100644 --- a/core/java/android/app/usage/NetworkStatsManager.java +++ b/core/java/android/app/usage/NetworkStatsManager.java @@ -263,20 +263,31 @@ public class NetworkStatsManager { /** * Query network usage statistics details for a given uid. * - * #see queryDetailsForUidTag(int, String, long, long, int, int) + * #see queryDetailsForUidTagState(int, String, long, long, int, int, int) */ public NetworkStats queryDetailsForUid(int networkType, String subscriberId, - long startTime, long endTime, int uid) throws SecurityException, RemoteException { - return queryDetailsForUidTag(networkType, subscriberId, startTime, endTime, uid, - NetworkStats.Bucket.TAG_NONE); + long startTime, long endTime, int uid) throws SecurityException { + return queryDetailsForUidTagState(networkType, subscriberId, startTime, endTime, uid, + NetworkStats.Bucket.TAG_NONE, NetworkStats.Bucket.STATE_ALL); } /** - * Query network usage statistics details for a given uid and tag. Only usable for uids - * belonging to calling user. Result is aggregated over state but not aggregated over time. - * This means buckets' start and end timestamps are going to be between 'startTime' and - * 'endTime' parameters. State is going to be {@link NetworkStats.Bucket#STATE_ALL}, uid the - * same as the 'uid' parameter and tag the same as 'tag' parameter. + * Query network usage statistics details for a given uid and tag. + * + * #see queryDetailsForUidTagState(int, String, long, long, int, int, int) + */ + public NetworkStats queryDetailsForUidTag(int networkType, String subscriberId, + long startTime, long endTime, int uid, int tag) throws SecurityException { + return queryDetailsForUidTagState(networkType, subscriberId, startTime, endTime, uid, + tag, NetworkStats.Bucket.STATE_ALL); + } + + /** + * Query network usage statistics details for a given uid, tag, and state. Only usable for uids + * belonging to calling user. Result is not aggregated over time. This means buckets' start and + * end timestamps are going to be between 'startTime' and 'endTime' parameters. The uid is going + * to be the same as the 'uid' parameter, the tag the same as the 'tag' parameter, and the state + * the same as the 'state' parameter. * defaultNetwork is going to be {@link NetworkStats.Bucket#DEFAULT_NETWORK_ALL}, * metered is going to be {@link NetworkStats.Bucket#METERED_ALL}, and * roaming is going to be {@link NetworkStats.Bucket#ROAMING_ALL}. @@ -297,17 +308,18 @@ public class NetworkStatsManager { * @return Statistics object or null if an error happened during statistics collection. * @throws SecurityException if permissions are insufficient to read network statistics. */ - public NetworkStats queryDetailsForUidTag(int networkType, String subscriberId, - long startTime, long endTime, int uid, int tag) throws SecurityException { + public NetworkStats queryDetailsForUidTagState(int networkType, String subscriberId, + long startTime, long endTime, int uid, int tag, int state) throws SecurityException { NetworkTemplate template; template = createTemplate(networkType, subscriberId); NetworkStats result; try { result = new NetworkStats(mContext, template, mFlags, startTime, endTime, mService); - result.startHistoryEnumeration(uid, tag); + result.startHistoryEnumeration(uid, tag, state); } catch (RemoteException e) { - Log.e(TAG, "Error while querying stats for uid=" + uid + " tag=" + tag, e); + Log.e(TAG, "Error while querying stats for uid=" + uid + " tag=" + tag + + " state=" + state, e); return null; } From 0023be9ea9e0bbfa15c1e833f7ea2c378fb5f1a1 Mon Sep 17 00:00:00 2001 From: Remi NGUYEN VAN Date: Tue, 27 Feb 2018 16:47:22 +0900 Subject: [PATCH 2/2] Fix network usage stats on 464xlat tethered. Usage stats corrections for 464xlat in NetworkStatsFactory are not applied to tethered traffic. Add adjustments in NetworkStatsService. After migrating external callers off NetworkStatsFactory, we will be able to only apply adjustments in NetworkStatsService and remove stacked interface tracking from NetworkStatsFactory. Bug: 72107146 Fixes: 72107146 Test: runtest frameworks-net & manual - checked corrected network usage Merged-In: Ieb25c41c651499fdd01225ae5ac21d95e3d823f5 Merged-In: I016722f3a0ae2ae0a1d48bfacc4fe07ee3578ef7 (cherry-pick of aosp I5ce450e616b4fddf21f2a491fe5d0c9e9f969bda) Change-Id: Id41cf22a0f9a63cb1832e9375bfb045861f08e52 --- core/java/android/net/NetworkStats.java | 75 +++++++++++++++++++ .../internal/net/NetworkStatsFactory.java | 60 ++++----------- .../server/net/NetworkStatsService.java | 8 +- 3 files changed, 92 insertions(+), 51 deletions(-) diff --git a/core/java/android/net/NetworkStats.java b/core/java/android/net/NetworkStats.java index 292bf8e196..f0dd2629f9 100644 --- a/core/java/android/net/NetworkStats.java +++ b/core/java/android/net/NetworkStats.java @@ -31,6 +31,7 @@ import java.io.CharArrayWriter; import java.io.PrintWriter; import java.util.Arrays; import java.util.HashSet; +import java.util.Map; import java.util.Objects; /** @@ -97,6 +98,11 @@ public class NetworkStats implements Parcelable { /** Denotes a request for stats at the interface and UID level. */ public static final int STATS_PER_UID = 1; + private static final String CLATD_INTERFACE_PREFIX = "v4-"; + // Delta between IPv4 header (20b) and IPv6 header (40b). + // Used for correct stats accounting on clatd interfaces. + private static final int IPV4V6_HEADER_DELTA = 20; + // TODO: move fields to "mVariable" notation /** @@ -758,6 +764,75 @@ public class NetworkStats implements Parcelable { return result; } + /** + * Calculate and apply adjustments to captured statistics for 464xlat traffic counted twice. + * + *

This mutates both base and stacked traffic stats, to account respectively for + * double-counted traffic and IPv4/IPv6 header size difference. + * + *

For 464xlat traffic, xt_qtaguid sees every IPv4 packet twice, once as a native IPv4 + * packet on the stacked interface, and once as translated to an IPv6 packet on the + * base interface. For correct stats accounting on the base interface, every 464xlat + * packet needs to be subtracted from the root UID on the base interface both for tx + * and rx traffic (http://b/12249687, http:/b/33681750). + * + *

This method will behave fine if {@code stackedIfaces} is an non-synchronized but add-only + * {@code ConcurrentHashMap} + * @param baseTraffic Traffic on the base interfaces. Will be mutated. + * @param stackedTraffic Stats with traffic stacked on top of our ifaces. Will also be mutated. + * @param stackedIfaces Mapping ipv6if -> ipv4if interface where traffic is counted on both. + */ + public static void apply464xlatAdjustments(NetworkStats baseTraffic, + NetworkStats stackedTraffic, Map stackedIfaces) { + // Total 464xlat traffic to subtract from uid 0 on all base interfaces. + // stackedIfaces may grow afterwards, but NetworkStats will just be resized automatically. + final NetworkStats adjustments = new NetworkStats(0, stackedIfaces.size()); + + // For recycling + Entry entry = null; + Entry adjust = new NetworkStats.Entry(IFACE_ALL, 0, 0, 0, 0, 0, 0, 0L, 0L, 0L, 0L, 0L); + + for (int i = 0; i < stackedTraffic.size; i++) { + entry = stackedTraffic.getValues(i, entry); + if (entry.iface == null || !entry.iface.startsWith(CLATD_INTERFACE_PREFIX)) { + continue; + } + final String baseIface = stackedIfaces.get(entry.iface); + if (baseIface == null) { + continue; + } + // Subtract any 464lat traffic seen for the root UID on the current base interface. + adjust.iface = baseIface; + adjust.rxBytes = -(entry.rxBytes + entry.rxPackets * IPV4V6_HEADER_DELTA); + adjust.txBytes = -(entry.txBytes + entry.txPackets * IPV4V6_HEADER_DELTA); + adjust.rxPackets = -entry.rxPackets; + adjust.txPackets = -entry.txPackets; + adjustments.combineValues(adjust); + + // For 464xlat traffic, xt_qtaguid only counts the bytes of the native IPv4 packet sent + // on the stacked interface with prefix "v4-" and drops the IPv6 header size after + // unwrapping. To account correctly for on-the-wire traffic, add the 20 additional bytes + // difference for all packets (http://b/12249687, http:/b/33681750). + entry.rxBytes += entry.rxPackets * IPV4V6_HEADER_DELTA; + entry.txBytes += entry.txPackets * IPV4V6_HEADER_DELTA; + stackedTraffic.setValues(i, entry); + } + + baseTraffic.combineAllValues(adjustments); + } + + /** + * Calculate and apply adjustments to captured statistics for 464xlat traffic counted twice. + * + *

This mutates the object this method is called on. Equivalent to calling + * {@link #apply464xlatAdjustments(NetworkStats, NetworkStats, Map)} with {@code this} as + * base and stacked traffic. + * @param stackedIfaces Mapping ipv6if -> ipv4if interface where traffic is counted on both. + */ + public void apply464xlatAdjustments(Map stackedIfaces) { + apply464xlatAdjustments(this, this, stackedIfaces); + } + /** * Return total statistics grouped by {@link #iface}; doesn't mutate the * original structure. diff --git a/core/java/com/android/internal/net/NetworkStatsFactory.java b/core/java/com/android/internal/net/NetworkStatsFactory.java index 0cf5375076..98afebc51b 100644 --- a/core/java/com/android/internal/net/NetworkStatsFactory.java +++ b/core/java/com/android/internal/net/NetworkStatsFactory.java @@ -56,11 +56,6 @@ public class NetworkStatsFactory { private static final boolean USE_NATIVE_PARSING = true; private static final boolean SANITY_CHECK_NATIVE = false; - private static final String CLATD_INTERFACE_PREFIX = "v4-"; - // Delta between IPv4 header (20b) and IPv6 header (40b). - // Used for correct stats accounting on clatd interfaces. - private static final int IPV4V6_HEADER_DELTA = 20; - /** Path to {@code /proc/net/dev}. */ private final File mStatsIfaceDev; /** Path to {@code /proc/net/xt_qtaguid/iface_stat_all}. */ @@ -96,7 +91,7 @@ public class NetworkStatsFactory { * {@link #noteStackedIface(String, String)}, but only interfaces noted before this method * is called are guaranteed to be included. */ - public static String[] augmentWithStackedInterfacesLocked(@Nullable String[] requiredIfaces) { + public static String[] augmentWithStackedInterfaces(@Nullable String[] requiredIfaces) { if (requiredIfaces == NetworkStats.INTERFACES_ALL) { return null; } @@ -118,6 +113,15 @@ public class NetworkStatsFactory { return relatedIfaces.toArray(outArray); } + /** + * Applies 464xlat adjustments with ifaces noted with {@link #noteStackedIface(String, String)}. + * @see NetworkStats#apply464xlatAdjustments(NetworkStats, NetworkStats, Map) + */ + public static void apply464xlatAdjustments(NetworkStats baseTraffic, + NetworkStats stackedTraffic) { + NetworkStats.apply464xlatAdjustments(baseTraffic, stackedTraffic, sStackedIfaces); + } + @VisibleForTesting public static void clearStackedIfaces() { sStackedIfaces.clear(); @@ -289,48 +293,10 @@ public class NetworkStatsFactory { NetworkStats lastStats) throws IOException { final NetworkStats stats = readNetworkStatsDetailInternal(limitUid, limitIfaces, limitTag, lastStats); - // Total 464xlat traffic to subtract from uid 0 on all base interfaces. - // sStackedIfaces may grow afterwards, but NetworkStats will just be resized automatically. - final NetworkStats adjustments = new NetworkStats(0, sStackedIfaces.size()); - NetworkStats.Entry entry = null; // For recycling - - // For 464xlat traffic, xt_qtaguid sees every IPv4 packet twice, once as a native IPv4 - // packet on the stacked interface, and once as translated to an IPv6 packet on the - // base interface. For correct stats accounting on the base interface, every 464xlat - // packet needs to be subtracted from the root UID on the base interface both for tx - // and rx traffic (http://b/12249687, http:/b/33681750). - for (int i = 0; i < stats.size(); i++) { - entry = stats.getValues(i, entry); - if (entry.iface == null || !entry.iface.startsWith(CLATD_INTERFACE_PREFIX)) { - continue; - } - final String baseIface = sStackedIfaces.get(entry.iface); - if (baseIface == null) { - continue; - } - - NetworkStats.Entry adjust = - new NetworkStats.Entry(baseIface, 0, 0, 0, 0, 0, 0, 0L, 0L, 0L, 0L, 0L); - // Subtract any 464lat traffic seen for the root UID on the current base interface. - adjust.rxBytes -= (entry.rxBytes + entry.rxPackets * IPV4V6_HEADER_DELTA); - adjust.txBytes -= (entry.txBytes + entry.txPackets * IPV4V6_HEADER_DELTA); - adjust.rxPackets -= entry.rxPackets; - adjust.txPackets -= entry.txPackets; - adjustments.combineValues(adjust); - - // For 464xlat traffic, xt_qtaguid only counts the bytes of the native IPv4 packet sent - // on the stacked interface with prefix "v4-" and drops the IPv6 header size after - // unwrapping. To account correctly for on-the-wire traffic, add the 20 additional bytes - // difference for all packets (http://b/12249687, http:/b/33681750). - entry.rxBytes = entry.rxPackets * IPV4V6_HEADER_DELTA; - entry.txBytes = entry.txPackets * IPV4V6_HEADER_DELTA; - entry.rxPackets = 0; - entry.txPackets = 0; - stats.combineValues(entry); - } - - stats.combineAllValues(adjustments); + // No locking here: apply464xlatAdjustments behaves fine with an add-only ConcurrentHashMap. + // TODO: remove this and only apply adjustments in NetworkStatsService. + stats.apply464xlatAdjustments(sStackedIfaces); return stats; } diff --git a/services/core/java/com/android/server/net/NetworkStatsService.java b/services/core/java/com/android/server/net/NetworkStatsService.java index 57828ca5dc..4e848f8f36 100644 --- a/services/core/java/com/android/server/net/NetworkStatsService.java +++ b/services/core/java/com/android/server/net/NetworkStatsService.java @@ -148,7 +148,6 @@ import java.time.ZoneOffset; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Map; /** * Collect and persist detailed network statistics, and provide this data to @@ -769,7 +768,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub { public NetworkStats getDetailedUidStats(String[] requiredIfaces) { try { final String[] ifacesToQuery = - NetworkStatsFactory.augmentWithStackedInterfacesLocked(requiredIfaces); + NetworkStatsFactory.augmentWithStackedInterfaces(requiredIfaces); return getNetworkStatsUidDetail(ifacesToQuery); } catch (RemoteException e) { Log.wtf(TAG, "Error compiling UID stats", e); @@ -1527,12 +1526,14 @@ public class NetworkStatsService extends INetworkStatsService.Stub { private NetworkStats getNetworkStatsUidDetail(String[] ifaces) throws RemoteException { + // TODO: remove 464xlat adjustments from NetworkStatsFactory and apply all at once here. final NetworkStats uidSnapshot = mNetworkManager.getNetworkStatsUidDetail(UID_ALL, ifaces); // fold tethering stats and operations into uid snapshot final NetworkStats tetherSnapshot = getNetworkStatsTethering(STATS_PER_UID); tetherSnapshot.filter(UID_ALL, ifaces, TAG_ALL); + NetworkStatsFactory.apply464xlatAdjustments(uidSnapshot, tetherSnapshot); uidSnapshot.combineAllValues(tetherSnapshot); final TelephonyManager telephonyManager = (TelephonyManager) mContext.getSystemService( @@ -1542,13 +1543,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub { final NetworkStats vtStats = telephonyManager.getVtDataUsage(STATS_PER_UID); if (vtStats != null) { vtStats.filter(UID_ALL, ifaces, TAG_ALL); + NetworkStatsFactory.apply464xlatAdjustments(uidSnapshot, vtStats); uidSnapshot.combineAllValues(vtStats); } uidSnapshot.combineAllValues(mUidOperations); - // TODO: apply tethering & VC 464xlat adjustments here - return uidSnapshot; }