LineageOS/android_packages_modules_Connectivity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ensure NetworkStats migrated snapshot is identical

Browse Source 
Read migration snapshot both from the platform API and the legacy
method, and Log.wtf if they are different. Use the legacy method data if
they are.

This ensures that using the migration API is a no-op, and that errors
are reported (through Log.wtf) if it would not have been if used as-is.

Ignore-AOSP-First: in a topic with internal-only changes
Test: NetworkStatsServiceTest
Bug: 230289468
Change-Id: I857ad18183d63d1aa16e89f89eb24009648720a2
This commit is contained in:
Remi NGUYEN VAN
2022-05-17 18:23:00 +09:00
committed by Junyu Lai
parent 27e0a9833e
commit 3776d862e2
3 changed files with 267 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
framework-t/src/android/net/NetworkStatsHistory.java
View File

@@ -32,6 +32,7 @@ import static android.text.format.DateUtils.SECOND_IN_MILLIS;
import static com.android.net.module.util.NetworkStatsUtils.multiplySafeByRational;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.compat.annotation.UnsupportedAppUsage;
import android.os.Build;
@@ -949,6 +950,25 @@ public final class NetworkStatsHistory implements Parcelable {
return writer.toString();
}
/**
* Same as "equals", but not actually called equals as this would affect public API behavior.
* @hide
*/
@Nullable
public boolean isSameAs(NetworkStatsHistory other) {
return bucketCount == other.bucketCount
&& Arrays.equals(bucketStart, other.bucketStart)
// Don't check activeTime since it can change on import due to the importer using
// recordHistory. It's also not exposed by the APIs or present in dumpsys or
// toString().
&& Arrays.equals(rxBytes, other.rxBytes)
&& Arrays.equals(rxPackets, other.rxPackets)
&& Arrays.equals(txBytes, other.txBytes)
&& Arrays.equals(txPackets, other.txPackets)
&& Arrays.equals(operations, other.operations)
&& totalBytes == other.totalBytes;
}
@UnsupportedAppUsage
public static final @android.annotation.NonNull Creator<NetworkStatsHistory> CREATOR = new Creator<NetworkStatsHistory>() {
@Override

169
service-t/src/com/android/server/net/NetworkStatsService.java
View File

@@ -76,6 +76,7 @@ import android.content.IntentFilter;
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.database.ContentObserver;
import android.net.ConnectivityManager;
import android.net.DataUsageRequest;
import android.net.INetd;
import android.net.INetworkStatsService;
@@ -573,6 +574,15 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
// TODO: Move more stuff into dependencies object.
@VisibleForTesting
public static class Dependencies {
/**
* Get legacy platform stats directory.
*/
@NonNull
public File getLegacyStatsDir() {
final File systemDataDir = new File(Environment.getDataDirectory(), "system");
return new File(systemDataDir, "netstats");
}
/**
* Get or create the directory that stores the persisted data usage.
*/
@@ -588,8 +598,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
statsDataDir = new File(apexDataDir, "netstats");
} else {
final File systemDataDir = new File(Environment.getDataDirectory(), "system");
statsDataDir = new File(systemDataDir, "netstats");
statsDataDir = getLegacyStatsDir();
}
if (statsDataDir.exists() || statsDataDir.mkdirs()) {
@@ -781,10 +790,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
mSystemReady = true;
// create data recorders along with historical rotators
mDevRecorder = buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false);
mXtRecorder = buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false);
mUidRecorder = buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false);
mUidTagRecorder = buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true);
mDevRecorder = buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false, mStatsDir);
mXtRecorder = buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false, mStatsDir);
mUidRecorder = buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false, mStatsDir);
mUidTagRecorder = buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true,
mStatsDir);
updatePersistThresholdsLocked();
@@ -848,11 +858,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
}
private NetworkStatsRecorder buildRecorder(
String prefix, NetworkStatsSettings.Config config, boolean includeTags) {
String prefix, NetworkStatsSettings.Config config, boolean includeTags,
File baseDir) {
final DropBoxManager dropBox = (DropBoxManager) mContext.getSystemService(
Context.DROPBOX_SERVICE);
return new NetworkStatsRecorder(new FileRotator(
mStatsDir, prefix, config.rotateAgeMillis, config.deleteAgeMillis),
baseDir, prefix, config.rotateAgeMillis, config.deleteAgeMillis),
mNonMonotonicObserver, dropBox, prefix, config.bucketDuration, includeTags);
}
@@ -921,16 +932,59 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
Log.i(TAG, "Starting import : attempts " + attempts + "/" + targetAttempts);
final List<MigrationInfo> migrations = List.of(
final MigrationInfo[] migrations = new MigrationInfo[]{
new MigrationInfo(mDevRecorder), new MigrationInfo(mXtRecorder),
new MigrationInfo(mUidRecorder), new MigrationInfo(mUidTagRecorder)
);
};
// Legacy directories will be created by recorders if they do not exist
final File legacyBaseDir = mDeps.getLegacyStatsDir();
final NetworkStatsRecorder[] legacyRecorders = new NetworkStatsRecorder[]{
buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false, legacyBaseDir),
buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false, legacyBaseDir),
buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false, legacyBaseDir),
buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true, legacyBaseDir)
};
long migrationEndTime = Long.MIN_VALUE;
boolean endedWithFallback = false;
try {
// First, read all legacy collections. This is OEM code and it can throw. Don't
// commit any data to disk until all are read.
for (final MigrationInfo migration : migrations) {
for (int i = 0; i < migrations.length; i++) {
final MigrationInfo migration = migrations[i];
migration.collection = readPlatformCollectionForRecorder(migration.recorder);
// Also read the collection with legacy method
final NetworkStatsRecorder legacyRecorder = legacyRecorders[i];
final NetworkStatsCollection legacyStats;
try {
legacyStats = legacyRecorder.getOrLoadCompleteLocked();
} catch (Throwable e) {
Log.wtf(TAG, "Failed to read stats with legacy method", e);
// Newer stats will be used here; that's the only thing that is usable
continue;
}
String errMsg;
Throwable exception = null;
try {
errMsg = compareStats(migration.collection, legacyStats);
} catch (Throwable e) {
errMsg = "Failed to compare migrated stats with all stats";
exception = e;
}
if (errMsg != null) {
Log.wtf(TAG, "NetworkStats import for migration " + i
+ " returned invalid data: " + errMsg, exception);
// Fall back to legacy stats for this boot. The stats for old data will be
// re-imported again on next boot until they succeed the import. This is fine
// since every import clears the previous stats for the imported timespan.
migration.collection = legacyStats;
endedWithFallback = true;
}
}
// Find the latest end time.
@@ -955,7 +1009,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
migration.recorder.importCollectionLocked(migration.collection);
}
Log.i(TAG, "Successfully imported platform collections");
if (endedWithFallback) {
Log.wtf(TAG, "Imported platform collections with legacy fallback");
} else {
Log.i(TAG, "Successfully imported platform collections");
}
} catch (Throwable e) {
// The code above calls OEM code that may behave differently across devices.
// It can throw any exception including RuntimeExceptions and
@@ -1004,6 +1062,93 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
}
}
private static String str(NetworkStatsCollection.Key key) {
StringBuilder sb = new StringBuilder()
.append(key.ident.toString())
.append(" uid=").append(key.uid);
if (key.set != SET_FOREGROUND) {
sb.append(" set=").append(key.set);
}
if (key.tag != 0) {
sb.append(" tag=").append(key.tag);
}
return sb.toString();
}
// The importer will modify some keys when importing them.
// In order to keep the comparison code simple, add such special cases here and simply
// ignore them. This should not impact fidelity much because the start/end checks and the total
// bytes check still need to pass.
private static boolean couldKeyChangeOnImport(NetworkStatsCollection.Key key) {
if (key.ident.isEmpty()) return false;
final NetworkIdentity firstIdent = key.ident.iterator().next();
// Non-mobile network with non-empty RAT type.
// This combination is invalid and the NetworkIdentity.Builder will throw if it is passed
// in, but it looks like it was previously possible to persist it to disk. The importer sets
// the RAT type to NETWORK_TYPE_ALL.
if (firstIdent.getType() != ConnectivityManager.TYPE_MOBILE
&& firstIdent.getRatType() != NetworkTemplate.NETWORK_TYPE_ALL) {
return true;
}
return false;
}
@Nullable
private static String compareStats(
NetworkStatsCollection migrated, NetworkStatsCollection legacy) {
final Map<NetworkStatsCollection.Key, NetworkStatsHistory> migEntries =
migrated.getEntries();
final Map<NetworkStatsCollection.Key, NetworkStatsHistory> legEntries = legacy.getEntries();
final ArraySet<NetworkStatsCollection.Key> unmatchedLegKeys =
new ArraySet<>(legEntries.keySet());
for (NetworkStatsCollection.Key legKey : legEntries.keySet()) {
final NetworkStatsHistory legHistory = legEntries.get(legKey);
final NetworkStatsHistory migHistory = migEntries.get(legKey);
if (migHistory == null && couldKeyChangeOnImport(legKey)) {
unmatchedLegKeys.remove(legKey);
continue;
}
if (migHistory == null) {
return "Missing migrated history for legacy key " + str(legKey)
+ ", legacy history was " + legHistory;
}
if (!migHistory.isSameAs(legHistory)) {
return "Difference in history for key " + legKey + "; legacy history " + legHistory
+ ", migrated history " + migHistory;
}
unmatchedLegKeys.remove(legKey);
}
if (!unmatchedLegKeys.isEmpty()) {
final NetworkStatsHistory first = legEntries.get(unmatchedLegKeys.valueAt(0));
return "Found unmatched legacy keys: count=" + unmatchedLegKeys.size()
+ ", first unmatched collection " + first;
}
if (migrated.getStartMillis() != legacy.getStartMillis()
|| migrated.getEndMillis() != legacy.getEndMillis()) {
return "Start / end of the collections "
+ migrated.getStartMillis() + "/" + legacy.getStartMillis() + " and "
+ migrated.getEndMillis() + "/" + legacy.getEndMillis()
+ " don't match";
}
if (migrated.getTotalBytes() != legacy.getTotalBytes()) {
return "Total bytes " + migrated.getTotalBytes() + " and " + legacy.getTotalBytes()
+ " don't match for collections with start/end "
+ migrated.getStartMillis()
+ "/" + legacy.getStartMillis();
}
return null;
}
@GuardedBy("mStatsLock")
@NonNull
private NetworkStatsCollection readPlatformCollectionForRecorder(

121
tests/unit/java/com/android/server/net/NetworkStatsServiceTest.java
View File

@@ -18,6 +18,7 @@ package com.android.server.net;
import static android.Manifest.permission.READ_NETWORK_USAGE_HISTORY;
import static android.Manifest.permission.UPDATE_DEVICE_STATS;
import static android.app.usage.NetworkStatsManager.PREFIX_DEV;
import static android.content.Intent.ACTION_UID_REMOVED;
import static android.content.Intent.EXTRA_UID;
import static android.content.pm.PackageManager.PERMISSION_DENIED;
@@ -56,6 +57,9 @@ import static android.net.NetworkTemplate.buildTemplateWifiWildcard;
import static android.net.TrafficStats.MB_IN_BYTES;
import static android.net.TrafficStats.UID_REMOVED;
import static android.net.TrafficStats.UID_TETHERING;
import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_UID;
import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_UID_TAG;
import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_XT;
import static android.text.format.DateUtils.DAY_IN_MILLIS;
import static android.text.format.DateUtils.HOUR_IN_MILLIS;
import static android.text.format.DateUtils.MINUTE_IN_MILLIS;
@@ -77,6 +81,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
@@ -94,7 +99,6 @@ import android.net.INetworkStatsSession;
import android.net.LinkProperties;
import android.net.Network;
import android.net.NetworkCapabilities;
import android.net.NetworkIdentity;
import android.net.NetworkStateSnapshot;
import android.net.NetworkStats;
import android.net.NetworkStatsCollection;
@@ -106,6 +110,7 @@ import android.net.TetheringManager;
import android.net.UnderlyingNetworkInfo;
import android.net.netstats.provider.INetworkStatsProviderCallback;
import android.net.wifi.WifiInfo;
import android.os.DropBoxManager;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.IBinder;
@@ -114,11 +119,13 @@ import android.os.SimpleClock;
import android.provider.Settings;
import android.system.ErrnoException;
import android.telephony.TelephonyManager;
import android.util.ArrayMap;
import androidx.annotation.Nullable;
import androidx.test.InstrumentationRegistry;
import androidx.test.filters.SmallTest;
import com.android.internal.util.FileRotator;
import com.android.internal.util.test.BroadcastInterceptingContext;
import com.android.net.module.util.IBpfMap;
import com.android.net.module.util.LocationPermissionChecker;
@@ -133,6 +140,16 @@ import com.android.testutils.HandlerUtils;
import com.android.testutils.TestBpfMap;
import com.android.testutils.TestableNetworkStatsProviderBinder;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Clock;
import java.time.ZoneOffset;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
import libcore.testing.io.TestIoUtils;
import org.junit.After;
@@ -144,15 +161,6 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import java.io.File;
import java.nio.file.Path;
import java.time.Clock;
import java.time.ZoneOffset;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Tests for {@link NetworkStatsService}.
*
@@ -191,6 +199,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
private long mElapsedRealtime;
private File mStatsDir;
private File mLegacyStatsDir;
private MockContext mServiceContext;
private @Mock TelephonyManager mTelephonyManager;
private static @Mock WifiInfo sWifiInfo;
@@ -224,8 +233,8 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
private ContentObserver mContentObserver;
private Handler mHandler;
private TetheringManager.TetheringEventCallback mTetheringEventCallback;
private NetworkStatsCollection mPlatformNetworkStatsCollection =
new NetworkStatsCollection(30 * HOUR_IN_MILLIS);
private Map<String, NetworkStatsCollection> mPlatformNetworkStatsCollection =
new ArrayMap<String, NetworkStatsCollection>();
private boolean mStoreFilesInApexData = false;
private int mImportLegacyTargetAttempts = 0;
private @Mock PersistentInt mImportLegacyAttemptsCounter;
@@ -296,6 +305,8 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
any(), any(), anyInt(), anyBoolean(), any())).thenReturn(true);
when(sWifiInfo.getNetworkKey()).thenReturn(TEST_WIFI_NETWORK_KEY);
mStatsDir = TestIoUtils.createTemporaryDirectory(getClass().getSimpleName());
mLegacyStatsDir = TestIoUtils.createTemporaryDirectory(
getClass().getSimpleName() + "-legacy");
PowerManager powerManager = (PowerManager) mServiceContext.getSystemService(
Context.POWER_SERVICE);
@@ -347,6 +358,11 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
@NonNull
private NetworkStatsService.Dependencies makeDependencies() {
return new NetworkStatsService.Dependencies() {
@Override
public File getLegacyStatsDir() {
return mLegacyStatsDir;
}
@Override
public File getOrCreateStatsDir() {
return mStatsDir;
@@ -377,7 +393,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
@Override
public NetworkStatsCollection readPlatformCollection(
@NonNull String prefix, long bucketDuration) {
return mPlatformNetworkStatsCollection;
return mPlatformNetworkStatsCollection.get(prefix);
}
@Override
@@ -1753,27 +1769,53 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
public void testDataMigration() throws Exception {
assertStatsFilesExist(false);
expectDefaultSettings();
final long bucketDuration = 30 * HOUR_IN_MILLIS;
final NetworkIdentity ident = new NetworkIdentity.Builder()
.setType(TYPE_MOBILE)
.setMetered(true)
.setSubscriberId(IMSI_1).build();
final NetworkStatsCollection.Key key = new NetworkStatsCollection.Key(
Set.of(ident), UID_ALL, SET_FOREGROUND, 0x0 /* tag */);
final NetworkStatsHistory history = new NetworkStatsHistory.Builder(bucketDuration, 0)
.addEntry(new NetworkStatsHistory.Entry(0, 10, 31, 3, 50, 5, 1)).build();
// Mock mobile traffic which will be reported by
// NetworkStatsDataMigrationUtils and verify it won't be absorbed if the flag is not set.
// TODO: Also mock UID traffic when service queries with PREFIX_UID. And
// verify with assertUidTotal.
mPlatformNetworkStatsCollection = new NetworkStatsCollection.Builder(bucketDuration)
.addEntry(key, history).build();
mStoreFilesInApexData = true;
mImportLegacyTargetAttempts = 0;
NetworkStateSnapshot[] states = new NetworkStateSnapshot[] {buildWifiState()};
mService.notifyNetworkStatus(NETWORKS_WIFI, states, getActiveIface(states),
new UnderlyingNetworkInfo[0]);
// modify some number on wifi, and trigger poll event
incrementCurrentTime(HOUR_IN_MILLIS);
// expectDefaultSettings();
expectNetworkStatsSummary(new NetworkStats(getElapsedRealtime(), 1)
.insertEntry(TEST_IFACE, 1024L, 8L, 2048L, 16L));
expectNetworkStatsUidDetail(new NetworkStats(getElapsedRealtime(), 2)
.insertEntry(TEST_IFACE, UID_RED, SET_DEFAULT, TAG_NONE, 512L, 4L, 256L, 2L, 0L)
.insertEntry(TEST_IFACE, UID_RED, SET_DEFAULT, 0xFAAD, 256L, 2L, 128L, 1L, 0L)
.insertEntry(TEST_IFACE, UID_RED, SET_FOREGROUND, TAG_NONE, 512L, 4L, 256L, 2L, 0L)
.insertEntry(TEST_IFACE, UID_RED, SET_FOREGROUND, 0xFAAD, 256L, 2L, 128L, 1L, 0L)
.insertEntry(TEST_IFACE, UID_BLUE, SET_DEFAULT, TAG_NONE, 128L, 1L, 128L, 1L, 0L));
mService.noteUidForeground(UID_RED, false);
verify(mUidCounterSetMap, never()).deleteEntry(any());
mService.incrementOperationCount(UID_RED, 0xFAAD, 4);
mService.noteUidForeground(UID_RED, true);
verify(mUidCounterSetMap).updateEntry(
eq(new U32(UID_RED)), eq(new U8((short) SET_FOREGROUND)));
mService.incrementOperationCount(UID_RED, 0xFAAD, 6);
forcePollAndWaitForIdle();
// Simulate shutdown to force persisting data
mServiceContext.sendBroadcast(new Intent(Intent.ACTION_SHUTDOWN));
assertStatsFilesExist(true);
// Move the files to the legacy directory to simulate an import from old data
for (File f : mStatsDir.listFiles()) {
Files.move(f.toPath(), mLegacyStatsDir.toPath().resolve(f.getName()));
}
assertStatsFilesExist(false);
// Fetch the stats from the legacy files and set platform stats collection to be identical
mPlatformNetworkStatsCollection.put(PREFIX_DEV,
getLegacyCollection(PREFIX_DEV, false /* includeTags */));
mPlatformNetworkStatsCollection.put(PREFIX_XT,
getLegacyCollection(PREFIX_XT, false /* includeTags */));
mPlatformNetworkStatsCollection.put(PREFIX_UID,
getLegacyCollection(PREFIX_UID, false /* includeTags */));
mPlatformNetworkStatsCollection.put(PREFIX_UID_TAG,
getLegacyCollection(PREFIX_UID_TAG, true /* includeTags */));
// Mock zero usage and boot through serviceReady(), verify there is no imported data.
expectDefaultSettings();
expectNetworkStatsUidDetail(buildEmptyStats());
@@ -1782,6 +1824,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
assertStatsFilesExist(false);
// Set the flag and reboot, verify the imported data is not there until next boot.
mStoreFilesInApexData = true;
mImportLegacyTargetAttempts = 3;
mServiceContext.sendBroadcast(new Intent(Intent.ACTION_SHUTDOWN));
assertStatsFilesExist(false);
@@ -1798,7 +1841,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
// 2. The imported data are persisted.
// 3. The attempts count is set to target attempts count to indicate a successful
// migration.
assertNetworkTotal(sTemplateImsi1, 31L, 3L, 50L, 5L, 1);
assertNetworkTotal(sTemplateWifi, 1024L, 8L, 2048L, 16L, 0);
assertStatsFilesExist(true);
verify(mImportLegacyAttemptsCounter).set(3);
verify(mImportLegacySuccessesCounter).set(1);
@@ -1807,6 +1850,22 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest {
// will decrease the retry counter by 1.
}
private NetworkStatsRecorder makeTestRecorder(File directory, String prefix, Config config,
boolean includeTags) {
final NetworkStats.NonMonotonicObserver observer =
mock(NetworkStats.NonMonotonicObserver.class);
final DropBoxManager dropBox = mock(DropBoxManager.class);
return new NetworkStatsRecorder(new FileRotator(
directory, prefix, config.rotateAgeMillis, config.deleteAgeMillis),
observer, dropBox, prefix, config.bucketDuration, includeTags);
}
private NetworkStatsCollection getLegacyCollection(String prefix, boolean includeTags) {
final NetworkStatsRecorder recorder = makeTestRecorder(mLegacyStatsDir, PREFIX_DEV,
mSettings.getDevConfig(), includeTags);
return recorder.getOrLoadCompleteLocked();
}
private void assertNetworkTotal(NetworkTemplate template, long rxBytes, long rxPackets,
long txBytes, long txPackets, int operations) throws Exception {
assertNetworkTotal(template, Long.MIN_VALUE, Long.MAX_VALUE, rxBytes, rxPackets, txBytes,