LineageOS/android_packages_modules_Connectivity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Correct proc file reader, optimizations.

Browse Source 
Moved away from BufferedReader, which only reads the first 8KB of
some proc files because it aggresively fills its buffer.  Optimized
proc parsing, now double the speed.  Tests to cover.

Log when NetworkStats counters roll backwards when subtracting, and
optimizations around findIndex().  When system removes UID, also
remove from last stats snapshot to avoid xt counters from rolling
backwards.

Bug: 5472949, 5458380
Change-Id: I07c08fe5233156fac2b84450f6291868bf9bfaf2
This commit is contained in:
Jeff Sharkey
2011-10-31 16:37:52 -07:00
parent 061658c2c7
commit cd5559a856
4 changed files with 155 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
core/java/android/net/NetworkStats.java
View File

@@ -16,10 +16,11 @@
package android.net;
import static com.android.internal.util.Preconditions.checkNotNull;
import android.os.Parcel;
import android.os.Parcelable;
import android.os.SystemClock;
import android.util.Log;
import android.util.SparseBooleanArray;
import com.android.internal.util.Objects;
@@ -54,6 +55,8 @@ public class NetworkStats implements Parcelable {
/** {@link #tag} value for total data across all tags. */
public static final int TAG_NONE = 0;
// TODO: move fields to "mVariable" notation
/**
* {@link SystemClock#elapsedRealtime()} timestamp when this data was
* generated.
@@ -295,8 +298,33 @@ public class NetworkStats implements Parcelable {
*/
public int findIndex(String iface, int uid, int set, int tag) {
for (int i = 0; i < size; i++) {
if (Objects.equal(iface, this.iface[i]) && uid == this.uid[i] && set == this.set[i]
&& tag == this.tag[i]) {
if (uid == this.uid[i] && set == this.set[i] && tag == this.tag[i]
&& Objects.equal(iface, this.iface[i])) {
return i;
}
}
return -1;
}
/**
* Find first stats index that matches the requested parameters, starting
* search around the hinted index as an optimization.
*/
// @VisibleForTesting
public int findIndexHinted(String iface, int uid, int set, int tag, int hintIndex) {
for (int offset = 0; offset < size; offset++) {
final int halfOffset = offset / 2;
// search outwards from hint index, alternating forward and backward
final int i;
if (offset % 2 == 0) {
i = (hintIndex + halfOffset) % size;
} else {
i = (size + hintIndex - halfOffset - 1) % size;
}
if (uid == this.uid[i] && set == this.set[i] && tag == this.tag[i]
&& Objects.equal(iface, this.iface[i])) {
return i;
}
}
@@ -423,40 +451,10 @@ public class NetworkStats implements Parcelable {
* Subtract the given {@link NetworkStats}, effectively leaving the delta
* between two snapshots in time. Assumes that statistics rows collect over
* time, and that none of them have disappeared.
*
* @throws IllegalArgumentException when given {@link NetworkStats} is
* non-monotonic.
*/
public NetworkStats subtract(NetworkStats value) {
return subtract(value, true, false);
}
/**
* Subtract the given {@link NetworkStats}, effectively leaving the delta
* between two snapshots in time. Assumes that statistics rows collect over
* time, and that none of them have disappeared.
* <p>
* Instead of throwing when counters are non-monotonic, this variant clamps
* results to never be negative.
*/
public NetworkStats subtractClamped(NetworkStats value) {
return subtract(value, false, true);
}
/**
* Subtract the given {@link NetworkStats}, effectively leaving the delta
* between two snapshots in time. Assumes that statistics rows collect over
* time, and that none of them have disappeared.
*
* @param enforceMonotonic Validate that incoming value is strictly
* monotonic compared to this object.
* @param clampNegative Instead of throwing like {@code enforceMonotonic},
* clamp resulting counters at 0 to prevent negative values.
*/
private NetworkStats subtract(
NetworkStats value, boolean enforceMonotonic, boolean clampNegative) {
public NetworkStats subtract(NetworkStats value) throws NonMonotonicException {
final long deltaRealtime = this.elapsedRealtime - value.elapsedRealtime;
if (enforceMonotonic && deltaRealtime < 0) {
if (deltaRealtime < 0) {
throw new IllegalArgumentException("found non-monotonic realtime");
}
@@ -470,7 +468,7 @@ public class NetworkStats implements Parcelable {
entry.tag = tag[i];
// find remote row that matches, and subtract
final int j = value.findIndex(entry.iface, entry.uid, entry.set, entry.tag);
final int j = value.findIndexHinted(entry.iface, entry.uid, entry.set, entry.tag, i);
if (j == -1) {
// newly appearing row, return entire value
entry.rxBytes = rxBytes[i];
@@ -485,20 +483,10 @@ public class NetworkStats implements Parcelable {
entry.txBytes = txBytes[i] - value.txBytes[j];
entry.txPackets = txPackets[i] - value.txPackets[j];
entry.operations = operations[i] - value.operations[j];
if (enforceMonotonic
&& (entry.rxBytes < 0 || entry.rxPackets < 0 || entry.txBytes < 0
|| entry.txPackets < 0 || entry.operations < 0)) {
Log.v(TAG, "lhs=" + this);
Log.v(TAG, "rhs=" + value);
throw new IllegalArgumentException(
"found non-monotonic values at lhs[" + i + "] - rhs[" + j + "]");
}
if (clampNegative) {
entry.rxBytes = Math.max(0, entry.rxBytes);
entry.rxPackets = Math.max(0, entry.rxPackets);
entry.txBytes = Math.max(0, entry.txBytes);
entry.txPackets = Math.max(0, entry.txPackets);
entry.operations = Math.max(0, entry.operations);
if (entry.rxBytes < 0 || entry.rxPackets < 0 || entry.txBytes < 0
|| entry.txPackets < 0 || entry.operations < 0) {
throw new NonMonotonicException(this, i, value, j);
}
}
@@ -564,6 +552,24 @@ public class NetworkStats implements Parcelable {
return stats;
}
/**
* Return all rows except those attributed to the requested UID; doesn't
* mutate the original structure.
*/
public NetworkStats withoutUid(int uid) {
final NetworkStats stats = new NetworkStats(elapsedRealtime, 10);
Entry entry = new Entry();
for (int i = 0; i < size; i++) {
entry = getValues(i, entry);
if (entry.uid != uid) {
stats.addValues(entry);
}
}
return stats;
}
public void dump(String prefix, PrintWriter pw) {
pw.print(prefix);
pw.print("NetworkStats: elapsedRealtime="); pw.println(elapsedRealtime);
@@ -625,4 +631,19 @@ public class NetworkStats implements Parcelable {
return new NetworkStats[size];
}
};
public static class NonMonotonicException extends Exception {
public final NetworkStats left;
public final NetworkStats right;
public final int leftIndex;
public final int rightIndex;
public NonMonotonicException(
NetworkStats left, int leftIndex, NetworkStats right, int rightIndex) {
this.left = checkNotNull(left, "missing left");
this.right = checkNotNull(right, "missing right");
this.leftIndex = leftIndex;
this.rightIndex = rightIndex;
}
}
}

8
core/java/android/net/TrafficStats.java
View File

@@ -20,6 +20,7 @@ import android.app.DownloadManager;
import android.app.backup.BackupManager;
import android.content.Context;
import android.media.MediaPlayer;
import android.net.NetworkStats.NonMonotonicException;
import android.os.RemoteException;
import android.os.ServiceManager;
@@ -192,12 +193,15 @@ public class TrafficStats {
throw new IllegalStateException("not profiling data");
}
try {
// subtract starting values and return delta
final NetworkStats profilingStop = getDataLayerSnapshotForUid(context);
final NetworkStats profilingDelta = profilingStop.subtractClamped(
sActiveProfilingStart);
final NetworkStats profilingDelta = profilingStop.subtract(sActiveProfilingStart);
sActiveProfilingStart = null;
return profilingDelta;
} catch (NonMonotonicException e) {
throw new RuntimeException(e);
}
}
}

66
core/java/com/android/internal/net/NetworkStatsFactory.java
View File

@@ -25,12 +25,14 @@ import android.net.NetworkStats;
import android.os.SystemClock;
import android.util.Slog;
import com.android.internal.util.ProcFileReader;
import com.google.android.collect.Lists;
import com.google.android.collect.Maps;
import com.google.android.collect.Sets;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
@@ -107,6 +109,7 @@ public class NetworkStatsFactory {
final NetworkStats stats = new NetworkStats(SystemClock.elapsedRealtime(), 6);
final NetworkStats.Entry entry = new NetworkStats.Entry();
// TODO: transition to ProcFileReader
// TODO: read directly from proc once headers are added
final ArrayList<String> keys = Lists.newArrayList(KEY_IFACE, KEY_ACTIVE, KEY_SNAP_RX_BYTES,
KEY_SNAP_RX_PACKETS, KEY_SNAP_TX_BYTES, KEY_SNAP_TX_PACKETS, KEY_RX_BYTES,
@@ -257,71 +260,58 @@ public class NetworkStatsFactory {
final NetworkStats stats = new NetworkStats(SystemClock.elapsedRealtime(), 24);
final NetworkStats.Entry entry = new NetworkStats.Entry();
// TODO: remove knownLines check once 5087722 verified
final HashSet<String> knownLines = Sets.newHashSet();
// TODO: remove lastIdx check once 5270106 verified
int lastIdx;
int idx = 1;
int lastIdx = 1;
final ArrayList<String> keys = Lists.newArrayList();
final ArrayList<String> values = Lists.newArrayList();
final HashMap<String, String> parsed = Maps.newHashMap();
BufferedReader reader = null;
String line = null;
ProcFileReader reader = null;
try {
reader = new BufferedReader(new FileReader(mStatsXtUid));
// open and consume header line
reader = new ProcFileReader(new FileInputStream(mStatsXtUid));
reader.finishLine();
// parse first line as header
line = reader.readLine();
splitLine(line, keys);
lastIdx = 1;
// parse remaining lines
while ((line = reader.readLine()) != null) {
splitLine(line, values);
parseLine(keys, values, parsed);
if (!knownLines.add(line)) {
throw new IllegalStateException("duplicate proc entry: " + line);
}
final int idx = getParsedInt(parsed, KEY_IDX);
while (reader.hasMoreData()) {
idx = reader.nextInt();
if (idx != lastIdx + 1) {
throw new IllegalStateException(
"inconsistent idx=" + idx + " after lastIdx=" + lastIdx);
}
lastIdx = idx;
entry.iface = parsed.get(KEY_IFACE);
entry.uid = getParsedInt(parsed, KEY_UID);
entry.set = getParsedInt(parsed, KEY_COUNTER_SET);
entry.tag = kernelToTag(parsed.get(KEY_TAG_HEX));
entry.rxBytes = getParsedLong(parsed, KEY_RX_BYTES);
entry.rxPackets = getParsedLong(parsed, KEY_RX_PACKETS);
entry.txBytes = getParsedLong(parsed, KEY_TX_BYTES);
entry.txPackets = getParsedLong(parsed, KEY_TX_PACKETS);
entry.iface = reader.nextString();
entry.tag = kernelToTag(reader.nextString());
entry.uid = reader.nextInt();
entry.set = reader.nextInt();
entry.rxBytes = reader.nextLong();
entry.rxPackets = reader.nextLong();
entry.txBytes = reader.nextLong();
entry.txPackets = reader.nextLong();
if (limitUid == UID_ALL || limitUid == entry.uid) {
stats.addValues(entry);
}
reader.finishLine();
}
} catch (NullPointerException e) {
throw new IllegalStateException("problem parsing line: " + line, e);
throw new IllegalStateException("problem parsing idx " + idx, e);
} catch (NumberFormatException e) {
throw new IllegalStateException("problem parsing line: " + line, e);
throw new IllegalStateException("problem parsing idx " + idx, e);
} catch (IOException e) {
throw new IllegalStateException("problem parsing line: " + line, e);
throw new IllegalStateException("problem parsing idx " + idx, e);
} finally {
IoUtils.closeQuietly(reader);
}
return stats;
}
@Deprecated
private static int getParsedInt(HashMap<String, String> parsed, String key) {
final String value = parsed.get(key);
return value != null ? Integer.parseInt(value) : 0;
}
@Deprecated
private static long getParsedLong(HashMap<String, String> parsed, String key) {
final String value = parsed.get(key);
return value != null ? Long.parseLong(value) : 0;
@@ -330,6 +320,7 @@ public class NetworkStatsFactory {
/**
* Split given line into {@link ArrayList}.
*/
@Deprecated
private static void splitLine(String line, ArrayList<String> outSplit) {
outSplit.clear();
@@ -343,6 +334,7 @@ public class NetworkStatsFactory {
* Zip the two given {@link ArrayList} as key and value pairs into
* {@link HashMap}.
*/
@Deprecated
private static void parseLine(
ArrayList<String> keys, ArrayList<String> values, HashMap<String, String> outParsed) {
outParsed.clear();

59
services/java/com/android/server/net/NetworkStatsService.java
View File

@@ -32,7 +32,6 @@ import static android.net.NetworkStats.SET_DEFAULT;
import static android.net.NetworkStats.SET_FOREGROUND;
import static android.net.NetworkStats.TAG_NONE;
import static android.net.NetworkStats.UID_ALL;
import static android.net.NetworkStatsHistory.randomLong;
import static android.net.NetworkTemplate.buildTemplateMobileAll;
import static android.net.NetworkTemplate.buildTemplateWifi;
import static android.net.TrafficStats.UID_REMOVED;
@@ -49,7 +48,6 @@ import static android.text.format.DateUtils.DAY_IN_MILLIS;
import static android.text.format.DateUtils.HOUR_IN_MILLIS;
import static android.text.format.DateUtils.MINUTE_IN_MILLIS;
import static android.text.format.DateUtils.SECOND_IN_MILLIS;
import static android.text.format.DateUtils.WEEK_IN_MILLIS;
import static com.android.internal.util.Preconditions.checkNotNull;
import static com.android.server.NetworkManagementService.LIMIT_GLOBAL_ALERT;
import static com.android.server.NetworkManagementSocketTagger.resetKernelUidStats;
@@ -73,9 +71,11 @@ import android.net.NetworkIdentity;
import android.net.NetworkInfo;
import android.net.NetworkState;
import android.net.NetworkStats;
import android.net.NetworkStats.NonMonotonicException;
import android.net.NetworkStatsHistory;
import android.net.NetworkTemplate;
import android.os.Binder;
import android.os.DropBoxManager;
import android.os.Environment;
import android.os.Handler;
import android.os.HandlerThread;
@@ -150,6 +150,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
/** Sample recent usage after each poll event. */
private static final boolean ENABLE_SAMPLE_AFTER_POLL = true;
private static final String TAG_NETSTATS_ERROR = "netstats_error";
private static final String DEV = "dev";
private static final String XT = "xt";
private static final String UID = "uid";
private final Context mContext;
private final INetworkManagementService mNetworkManager;
private final IAlarmManager mAlarmManager;
@@ -160,6 +166,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
private final PowerManager.WakeLock mWakeLock;
private IConnectivityManager mConnManager;
private DropBoxManager mDropBox;
// @VisibleForTesting
public static final String ACTION_NETWORK_STATS_POLL =
@@ -306,6 +313,8 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
// bootstrap initial stats to prevent double-counting later
bootstrapStats();
mDropBox = (DropBoxManager) mContext.getSystemService(Context.DROPBOX_SERVICE);
}
private void shutdownLocked() {
@@ -621,7 +630,6 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
// broadcast.
final int uid = intent.getIntExtra(EXTRA_UID, 0);
synchronized (mStatsLock) {
// TODO: perform one last stats poll for UID
mWakeLock.acquire();
try {
removeUidLocked(uid);
@@ -829,9 +837,9 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
// persist when enough network data has occurred
final long persistNetworkDevDelta = computeStatsDelta(
mLastPersistNetworkDevSnapshot, networkDevSnapshot, true).getTotalBytes();
mLastPersistNetworkDevSnapshot, networkDevSnapshot, true, DEV).getTotalBytes();
final long persistNetworkXtDelta = computeStatsDelta(
mLastPersistNetworkXtSnapshot, networkXtSnapshot, true).getTotalBytes();
mLastPersistNetworkXtSnapshot, networkXtSnapshot, true, XT).getTotalBytes();
final boolean networkOverThreshold = persistNetworkDevDelta > threshold
|| persistNetworkXtDelta > threshold;
if (persistForce || (persistNetwork && networkOverThreshold)) {
@@ -842,8 +850,8 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
}
// persist when enough uid data has occurred
final long persistUidDelta = computeStatsDelta(mLastPersistUidSnapshot, uidSnapshot, true)
.getTotalBytes();
final long persistUidDelta = computeStatsDelta(
mLastPersistUidSnapshot, uidSnapshot, true, UID).getTotalBytes();
if (persistForce || (persistUid && persistUidDelta > threshold)) {
writeUidStatsLocked();
mLastPersistUidSnapshot = uidSnapshot;
@@ -872,7 +880,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
final HashSet<String> unknownIface = Sets.newHashSet();
final NetworkStats delta = computeStatsDelta(
mLastPollNetworkDevSnapshot, networkDevSnapshot, false);
mLastPollNetworkDevSnapshot, networkDevSnapshot, false, DEV);
final long timeStart = currentTime - delta.getElapsedRealtime();
NetworkStats.Entry entry = null;
@@ -902,7 +910,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
final HashSet<String> unknownIface = Sets.newHashSet();
final NetworkStats delta = computeStatsDelta(
mLastPollNetworkXtSnapshot, networkXtSnapshot, false);
mLastPollNetworkXtSnapshot, networkXtSnapshot, false, XT);
final long timeStart = currentTime - delta.getElapsedRealtime();
NetworkStats.Entry entry = null;
@@ -931,9 +939,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
private void performUidPollLocked(NetworkStats uidSnapshot, long currentTime) {
ensureUidStatsLoadedLocked();
final NetworkStats delta = computeStatsDelta(mLastPollUidSnapshot, uidSnapshot, false);
final NetworkStats delta = computeStatsDelta(
mLastPollUidSnapshot, uidSnapshot, false, UID);
final NetworkStats operationsDelta = computeStatsDelta(
mLastPollOperationsSnapshot, mOperations, false);
mLastPollOperationsSnapshot, mOperations, false, UID);
final long timeStart = currentTime - delta.getElapsedRealtime();
NetworkStats.Entry entry = null;
@@ -1014,6 +1023,9 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
private void removeUidLocked(int uid) {
ensureUidStatsLoadedLocked();
// perform one last poll before removing
performPollLocked(FLAG_PERSIST_ALL);
final ArrayList<UidStatsKey> knownKeys = Lists.newArrayList();
knownKeys.addAll(mUidStats.keySet());
@@ -1031,6 +1043,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
}
}
// clear UID from current stats snapshot
mLastPollUidSnapshot = mLastPollUidSnapshot.withoutUid(uid);
mLastPollNetworkXtSnapshot = computeNetworkXtSnapshotFromUid(mLastPollUidSnapshot);
// clear kernel stats associated with UID
resetKernelUidStats(uid);
@@ -1490,10 +1506,25 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
* Return the delta between two {@link NetworkStats} snapshots, where {@code
* before} can be {@code null}.
*/
private static NetworkStats computeStatsDelta(
NetworkStats before, NetworkStats current, boolean collectStale) {
private NetworkStats computeStatsDelta(
NetworkStats before, NetworkStats current, boolean collectStale, String type) {
if (before != null) {
return current.subtractClamped(before);
try {
return current.subtract(before);
} catch (NonMonotonicException e) {
Log.w(TAG, "found non-monotonic values; saving to dropbox");
// record error for debugging
final StringBuilder builder = new StringBuilder();
builder.append("found non-monotonic " + type + "values at left[" + e.leftIndex
+ "] - right[" + e.rightIndex + "]\n");
builder.append("left=").append(e.left).append('\n');
builder.append("right=").append(e.right).append('\n');
mDropBox.addText(TAG_NETSTATS_ERROR, builder.toString());
// return empty delta to avoid recording broken stats
return new NetworkStats(0L, 10);
}
} else if (collectStale) {
// caller is okay collecting stale stats for first call.
return current;