diff --git a/framework-t/src/android/net/NetworkStatsHistory.java b/framework-t/src/android/net/NetworkStatsHistory.java index b45d44dec1..60dad905b2 100644 --- a/framework-t/src/android/net/NetworkStatsHistory.java +++ b/framework-t/src/android/net/NetworkStatsHistory.java @@ -32,6 +32,7 @@ import static android.text.format.DateUtils.SECOND_IN_MILLIS; import static com.android.net.module.util.NetworkStatsUtils.multiplySafeByRational; import android.annotation.NonNull; +import android.annotation.Nullable; import android.annotation.SystemApi; import android.compat.annotation.UnsupportedAppUsage; import android.os.Build; @@ -949,6 +950,25 @@ public final class NetworkStatsHistory implements Parcelable { return writer.toString(); } + /** + * Same as "equals", but not actually called equals as this would affect public API behavior. + * @hide + */ + @Nullable + public boolean isSameAs(NetworkStatsHistory other) { + return bucketCount == other.bucketCount + && Arrays.equals(bucketStart, other.bucketStart) + // Don't check activeTime since it can change on import due to the importer using + // recordHistory. It's also not exposed by the APIs or present in dumpsys or + // toString(). + && Arrays.equals(rxBytes, other.rxBytes) + && Arrays.equals(rxPackets, other.rxPackets) + && Arrays.equals(txBytes, other.txBytes) + && Arrays.equals(txPackets, other.txPackets) + && Arrays.equals(operations, other.operations) + && totalBytes == other.totalBytes; + } + @UnsupportedAppUsage public static final @android.annotation.NonNull Creator CREATOR = new Creator() { @Override diff --git a/service-t/src/com/android/server/net/NetworkStatsService.java b/service-t/src/com/android/server/net/NetworkStatsService.java index 61e22a11ec..892c0b6f1c 100644 --- a/service-t/src/com/android/server/net/NetworkStatsService.java +++ b/service-t/src/com/android/server/net/NetworkStatsService.java @@ -76,6 +76,7 @@ import android.content.IntentFilter; import android.content.pm.ApplicationInfo; import android.content.pm.PackageManager; import android.database.ContentObserver; +import android.net.ConnectivityManager; import android.net.DataUsageRequest; import android.net.INetd; import android.net.INetworkStatsService; @@ -573,6 +574,15 @@ public class NetworkStatsService extends INetworkStatsService.Stub { // TODO: Move more stuff into dependencies object. @VisibleForTesting public static class Dependencies { + /** + * Get legacy platform stats directory. + */ + @NonNull + public File getLegacyStatsDir() { + final File systemDataDir = new File(Environment.getDataDirectory(), "system"); + return new File(systemDataDir, "netstats"); + } + /** * Get or create the directory that stores the persisted data usage. */ @@ -588,8 +598,7 @@ public class NetworkStatsService extends INetworkStatsService.Stub { statsDataDir = new File(apexDataDir, "netstats"); } else { - final File systemDataDir = new File(Environment.getDataDirectory(), "system"); - statsDataDir = new File(systemDataDir, "netstats"); + statsDataDir = getLegacyStatsDir(); } if (statsDataDir.exists() || statsDataDir.mkdirs()) { @@ -781,10 +790,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub { mSystemReady = true; // create data recorders along with historical rotators - mDevRecorder = buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false); - mXtRecorder = buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false); - mUidRecorder = buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false); - mUidTagRecorder = buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true); + mDevRecorder = buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false, mStatsDir); + mXtRecorder = buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false, mStatsDir); + mUidRecorder = buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false, mStatsDir); + mUidTagRecorder = buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true, + mStatsDir); updatePersistThresholdsLocked(); @@ -848,11 +858,12 @@ public class NetworkStatsService extends INetworkStatsService.Stub { } private NetworkStatsRecorder buildRecorder( - String prefix, NetworkStatsSettings.Config config, boolean includeTags) { + String prefix, NetworkStatsSettings.Config config, boolean includeTags, + File baseDir) { final DropBoxManager dropBox = (DropBoxManager) mContext.getSystemService( Context.DROPBOX_SERVICE); return new NetworkStatsRecorder(new FileRotator( - mStatsDir, prefix, config.rotateAgeMillis, config.deleteAgeMillis), + baseDir, prefix, config.rotateAgeMillis, config.deleteAgeMillis), mNonMonotonicObserver, dropBox, prefix, config.bucketDuration, includeTags); } @@ -921,16 +932,59 @@ public class NetworkStatsService extends INetworkStatsService.Stub { Log.i(TAG, "Starting import : attempts " + attempts + "/" + targetAttempts); - final List migrations = List.of( + final MigrationInfo[] migrations = new MigrationInfo[]{ new MigrationInfo(mDevRecorder), new MigrationInfo(mXtRecorder), new MigrationInfo(mUidRecorder), new MigrationInfo(mUidTagRecorder) - ); + }; + + // Legacy directories will be created by recorders if they do not exist + final File legacyBaseDir = mDeps.getLegacyStatsDir(); + final NetworkStatsRecorder[] legacyRecorders = new NetworkStatsRecorder[]{ + buildRecorder(PREFIX_DEV, mSettings.getDevConfig(), false, legacyBaseDir), + buildRecorder(PREFIX_XT, mSettings.getXtConfig(), false, legacyBaseDir), + buildRecorder(PREFIX_UID, mSettings.getUidConfig(), false, legacyBaseDir), + buildRecorder(PREFIX_UID_TAG, mSettings.getUidTagConfig(), true, legacyBaseDir) + }; + long migrationEndTime = Long.MIN_VALUE; + boolean endedWithFallback = false; try { // First, read all legacy collections. This is OEM code and it can throw. Don't // commit any data to disk until all are read. - for (final MigrationInfo migration : migrations) { + for (int i = 0; i < migrations.length; i++) { + final MigrationInfo migration = migrations[i]; migration.collection = readPlatformCollectionForRecorder(migration.recorder); + + // Also read the collection with legacy method + final NetworkStatsRecorder legacyRecorder = legacyRecorders[i]; + + final NetworkStatsCollection legacyStats; + try { + legacyStats = legacyRecorder.getOrLoadCompleteLocked(); + } catch (Throwable e) { + Log.wtf(TAG, "Failed to read stats with legacy method", e); + // Newer stats will be used here; that's the only thing that is usable + continue; + } + + String errMsg; + Throwable exception = null; + try { + errMsg = compareStats(migration.collection, legacyStats); + } catch (Throwable e) { + errMsg = "Failed to compare migrated stats with all stats"; + exception = e; + } + + if (errMsg != null) { + Log.wtf(TAG, "NetworkStats import for migration " + i + + " returned invalid data: " + errMsg, exception); + // Fall back to legacy stats for this boot. The stats for old data will be + // re-imported again on next boot until they succeed the import. This is fine + // since every import clears the previous stats for the imported timespan. + migration.collection = legacyStats; + endedWithFallback = true; + } } // Find the latest end time. @@ -955,7 +1009,11 @@ public class NetworkStatsService extends INetworkStatsService.Stub { migration.recorder.importCollectionLocked(migration.collection); } - Log.i(TAG, "Successfully imported platform collections"); + if (endedWithFallback) { + Log.wtf(TAG, "Imported platform collections with legacy fallback"); + } else { + Log.i(TAG, "Successfully imported platform collections"); + } } catch (Throwable e) { // The code above calls OEM code that may behave differently across devices. // It can throw any exception including RuntimeExceptions and @@ -1004,6 +1062,93 @@ public class NetworkStatsService extends INetworkStatsService.Stub { } } + private static String str(NetworkStatsCollection.Key key) { + StringBuilder sb = new StringBuilder() + .append(key.ident.toString()) + .append(" uid=").append(key.uid); + if (key.set != SET_FOREGROUND) { + sb.append(" set=").append(key.set); + } + if (key.tag != 0) { + sb.append(" tag=").append(key.tag); + } + return sb.toString(); + } + + // The importer will modify some keys when importing them. + // In order to keep the comparison code simple, add such special cases here and simply + // ignore them. This should not impact fidelity much because the start/end checks and the total + // bytes check still need to pass. + private static boolean couldKeyChangeOnImport(NetworkStatsCollection.Key key) { + if (key.ident.isEmpty()) return false; + final NetworkIdentity firstIdent = key.ident.iterator().next(); + + // Non-mobile network with non-empty RAT type. + // This combination is invalid and the NetworkIdentity.Builder will throw if it is passed + // in, but it looks like it was previously possible to persist it to disk. The importer sets + // the RAT type to NETWORK_TYPE_ALL. + if (firstIdent.getType() != ConnectivityManager.TYPE_MOBILE + && firstIdent.getRatType() != NetworkTemplate.NETWORK_TYPE_ALL) { + return true; + } + + return false; + } + + @Nullable + private static String compareStats( + NetworkStatsCollection migrated, NetworkStatsCollection legacy) { + final Map migEntries = + migrated.getEntries(); + final Map legEntries = legacy.getEntries(); + + final ArraySet unmatchedLegKeys = + new ArraySet<>(legEntries.keySet()); + + for (NetworkStatsCollection.Key legKey : legEntries.keySet()) { + final NetworkStatsHistory legHistory = legEntries.get(legKey); + final NetworkStatsHistory migHistory = migEntries.get(legKey); + + if (migHistory == null && couldKeyChangeOnImport(legKey)) { + unmatchedLegKeys.remove(legKey); + continue; + } + + if (migHistory == null) { + return "Missing migrated history for legacy key " + str(legKey) + + ", legacy history was " + legHistory; + } + if (!migHistory.isSameAs(legHistory)) { + return "Difference in history for key " + legKey + "; legacy history " + legHistory + + ", migrated history " + migHistory; + } + unmatchedLegKeys.remove(legKey); + } + + if (!unmatchedLegKeys.isEmpty()) { + final NetworkStatsHistory first = legEntries.get(unmatchedLegKeys.valueAt(0)); + return "Found unmatched legacy keys: count=" + unmatchedLegKeys.size() + + ", first unmatched collection " + first; + } + + if (migrated.getStartMillis() != legacy.getStartMillis() + || migrated.getEndMillis() != legacy.getEndMillis()) { + return "Start / end of the collections " + + migrated.getStartMillis() + "/" + legacy.getStartMillis() + " and " + + migrated.getEndMillis() + "/" + legacy.getEndMillis() + + " don't match"; + } + + if (migrated.getTotalBytes() != legacy.getTotalBytes()) { + return "Total bytes " + migrated.getTotalBytes() + " and " + legacy.getTotalBytes() + + " don't match for collections with start/end " + + migrated.getStartMillis() + + "/" + legacy.getStartMillis(); + } + + return null; + } + @GuardedBy("mStatsLock") @NonNull private NetworkStatsCollection readPlatformCollectionForRecorder( diff --git a/tests/unit/java/com/android/server/net/NetworkStatsServiceTest.java b/tests/unit/java/com/android/server/net/NetworkStatsServiceTest.java index 24731bfa41..f1820b3acb 100644 --- a/tests/unit/java/com/android/server/net/NetworkStatsServiceTest.java +++ b/tests/unit/java/com/android/server/net/NetworkStatsServiceTest.java @@ -18,6 +18,7 @@ package com.android.server.net; import static android.Manifest.permission.READ_NETWORK_USAGE_HISTORY; import static android.Manifest.permission.UPDATE_DEVICE_STATS; +import static android.app.usage.NetworkStatsManager.PREFIX_DEV; import static android.content.Intent.ACTION_UID_REMOVED; import static android.content.Intent.EXTRA_UID; import static android.content.pm.PackageManager.PERMISSION_DENIED; @@ -56,6 +57,9 @@ import static android.net.NetworkTemplate.buildTemplateWifiWildcard; import static android.net.TrafficStats.MB_IN_BYTES; import static android.net.TrafficStats.UID_REMOVED; import static android.net.TrafficStats.UID_TETHERING; +import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_UID; +import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_UID_TAG; +import static android.net.netstats.NetworkStatsDataMigrationUtils.PREFIX_XT; import static android.text.format.DateUtils.DAY_IN_MILLIS; import static android.text.format.DateUtils.HOUR_IN_MILLIS; import static android.text.format.DateUtils.MINUTE_IN_MILLIS; @@ -77,6 +81,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; @@ -94,7 +99,6 @@ import android.net.INetworkStatsSession; import android.net.LinkProperties; import android.net.Network; import android.net.NetworkCapabilities; -import android.net.NetworkIdentity; import android.net.NetworkStateSnapshot; import android.net.NetworkStats; import android.net.NetworkStatsCollection; @@ -106,6 +110,7 @@ import android.net.TetheringManager; import android.net.UnderlyingNetworkInfo; import android.net.netstats.provider.INetworkStatsProviderCallback; import android.net.wifi.WifiInfo; +import android.os.DropBoxManager; import android.os.Handler; import android.os.HandlerThread; import android.os.IBinder; @@ -114,11 +119,13 @@ import android.os.SimpleClock; import android.provider.Settings; import android.system.ErrnoException; import android.telephony.TelephonyManager; +import android.util.ArrayMap; import androidx.annotation.Nullable; import androidx.test.InstrumentationRegistry; import androidx.test.filters.SmallTest; +import com.android.internal.util.FileRotator; import com.android.internal.util.test.BroadcastInterceptingContext; import com.android.net.module.util.IBpfMap; import com.android.net.module.util.LocationPermissionChecker; @@ -133,6 +140,16 @@ import com.android.testutils.HandlerUtils; import com.android.testutils.TestBpfMap; import com.android.testutils.TestableNetworkStatsProviderBinder; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.ZoneOffset; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + import libcore.testing.io.TestIoUtils; import org.junit.After; @@ -144,15 +161,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.io.File; -import java.nio.file.Path; -import java.time.Clock; -import java.time.ZoneOffset; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - /** * Tests for {@link NetworkStatsService}. * @@ -191,6 +199,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { private long mElapsedRealtime; private File mStatsDir; + private File mLegacyStatsDir; private MockContext mServiceContext; private @Mock TelephonyManager mTelephonyManager; private static @Mock WifiInfo sWifiInfo; @@ -224,8 +233,8 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { private ContentObserver mContentObserver; private Handler mHandler; private TetheringManager.TetheringEventCallback mTetheringEventCallback; - private NetworkStatsCollection mPlatformNetworkStatsCollection = - new NetworkStatsCollection(30 * HOUR_IN_MILLIS); + private Map mPlatformNetworkStatsCollection = + new ArrayMap(); private boolean mStoreFilesInApexData = false; private int mImportLegacyTargetAttempts = 0; private @Mock PersistentInt mImportLegacyAttemptsCounter; @@ -296,6 +305,8 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { any(), any(), anyInt(), anyBoolean(), any())).thenReturn(true); when(sWifiInfo.getNetworkKey()).thenReturn(TEST_WIFI_NETWORK_KEY); mStatsDir = TestIoUtils.createTemporaryDirectory(getClass().getSimpleName()); + mLegacyStatsDir = TestIoUtils.createTemporaryDirectory( + getClass().getSimpleName() + "-legacy"); PowerManager powerManager = (PowerManager) mServiceContext.getSystemService( Context.POWER_SERVICE); @@ -347,6 +358,11 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { @NonNull private NetworkStatsService.Dependencies makeDependencies() { return new NetworkStatsService.Dependencies() { + @Override + public File getLegacyStatsDir() { + return mLegacyStatsDir; + } + @Override public File getOrCreateStatsDir() { return mStatsDir; @@ -377,7 +393,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { @Override public NetworkStatsCollection readPlatformCollection( @NonNull String prefix, long bucketDuration) { - return mPlatformNetworkStatsCollection; + return mPlatformNetworkStatsCollection.get(prefix); } @Override @@ -1753,27 +1769,53 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { public void testDataMigration() throws Exception { assertStatsFilesExist(false); expectDefaultSettings(); - final long bucketDuration = 30 * HOUR_IN_MILLIS; - final NetworkIdentity ident = new NetworkIdentity.Builder() - .setType(TYPE_MOBILE) - .setMetered(true) - .setSubscriberId(IMSI_1).build(); - final NetworkStatsCollection.Key key = new NetworkStatsCollection.Key( - Set.of(ident), UID_ALL, SET_FOREGROUND, 0x0 /* tag */); - final NetworkStatsHistory history = new NetworkStatsHistory.Builder(bucketDuration, 0) - .addEntry(new NetworkStatsHistory.Entry(0, 10, 31, 3, 50, 5, 1)).build(); - // Mock mobile traffic which will be reported by - // NetworkStatsDataMigrationUtils and verify it won't be absorbed if the flag is not set. - // TODO: Also mock UID traffic when service queries with PREFIX_UID. And - // verify with assertUidTotal. - mPlatformNetworkStatsCollection = new NetworkStatsCollection.Builder(bucketDuration) - .addEntry(key, history).build(); - mStoreFilesInApexData = true; - mImportLegacyTargetAttempts = 0; + NetworkStateSnapshot[] states = new NetworkStateSnapshot[] {buildWifiState()}; + + mService.notifyNetworkStatus(NETWORKS_WIFI, states, getActiveIface(states), + new UnderlyingNetworkInfo[0]); + + // modify some number on wifi, and trigger poll event + incrementCurrentTime(HOUR_IN_MILLIS); + // expectDefaultSettings(); + expectNetworkStatsSummary(new NetworkStats(getElapsedRealtime(), 1) + .insertEntry(TEST_IFACE, 1024L, 8L, 2048L, 16L)); + expectNetworkStatsUidDetail(new NetworkStats(getElapsedRealtime(), 2) + .insertEntry(TEST_IFACE, UID_RED, SET_DEFAULT, TAG_NONE, 512L, 4L, 256L, 2L, 0L) + .insertEntry(TEST_IFACE, UID_RED, SET_DEFAULT, 0xFAAD, 256L, 2L, 128L, 1L, 0L) + .insertEntry(TEST_IFACE, UID_RED, SET_FOREGROUND, TAG_NONE, 512L, 4L, 256L, 2L, 0L) + .insertEntry(TEST_IFACE, UID_RED, SET_FOREGROUND, 0xFAAD, 256L, 2L, 128L, 1L, 0L) + .insertEntry(TEST_IFACE, UID_BLUE, SET_DEFAULT, TAG_NONE, 128L, 1L, 128L, 1L, 0L)); + + mService.noteUidForeground(UID_RED, false); + verify(mUidCounterSetMap, never()).deleteEntry(any()); + mService.incrementOperationCount(UID_RED, 0xFAAD, 4); + mService.noteUidForeground(UID_RED, true); + verify(mUidCounterSetMap).updateEntry( + eq(new U32(UID_RED)), eq(new U8((short) SET_FOREGROUND))); + mService.incrementOperationCount(UID_RED, 0xFAAD, 6); + + forcePollAndWaitForIdle(); + // Simulate shutdown to force persisting data mServiceContext.sendBroadcast(new Intent(Intent.ACTION_SHUTDOWN)); + assertStatsFilesExist(true); + + // Move the files to the legacy directory to simulate an import from old data + for (File f : mStatsDir.listFiles()) { + Files.move(f.toPath(), mLegacyStatsDir.toPath().resolve(f.getName())); + } assertStatsFilesExist(false); + // Fetch the stats from the legacy files and set platform stats collection to be identical + mPlatformNetworkStatsCollection.put(PREFIX_DEV, + getLegacyCollection(PREFIX_DEV, false /* includeTags */)); + mPlatformNetworkStatsCollection.put(PREFIX_XT, + getLegacyCollection(PREFIX_XT, false /* includeTags */)); + mPlatformNetworkStatsCollection.put(PREFIX_UID, + getLegacyCollection(PREFIX_UID, false /* includeTags */)); + mPlatformNetworkStatsCollection.put(PREFIX_UID_TAG, + getLegacyCollection(PREFIX_UID_TAG, true /* includeTags */)); + // Mock zero usage and boot through serviceReady(), verify there is no imported data. expectDefaultSettings(); expectNetworkStatsUidDetail(buildEmptyStats()); @@ -1782,6 +1824,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { assertStatsFilesExist(false); // Set the flag and reboot, verify the imported data is not there until next boot. + mStoreFilesInApexData = true; mImportLegacyTargetAttempts = 3; mServiceContext.sendBroadcast(new Intent(Intent.ACTION_SHUTDOWN)); assertStatsFilesExist(false); @@ -1798,7 +1841,7 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { // 2. The imported data are persisted. // 3. The attempts count is set to target attempts count to indicate a successful // migration. - assertNetworkTotal(sTemplateImsi1, 31L, 3L, 50L, 5L, 1); + assertNetworkTotal(sTemplateWifi, 1024L, 8L, 2048L, 16L, 0); assertStatsFilesExist(true); verify(mImportLegacyAttemptsCounter).set(3); verify(mImportLegacySuccessesCounter).set(1); @@ -1807,6 +1850,22 @@ public class NetworkStatsServiceTest extends NetworkStatsBaseTest { // will decrease the retry counter by 1. } + private NetworkStatsRecorder makeTestRecorder(File directory, String prefix, Config config, + boolean includeTags) { + final NetworkStats.NonMonotonicObserver observer = + mock(NetworkStats.NonMonotonicObserver.class); + final DropBoxManager dropBox = mock(DropBoxManager.class); + return new NetworkStatsRecorder(new FileRotator( + directory, prefix, config.rotateAgeMillis, config.deleteAgeMillis), + observer, dropBox, prefix, config.bucketDuration, includeTags); + } + + private NetworkStatsCollection getLegacyCollection(String prefix, boolean includeTags) { + final NetworkStatsRecorder recorder = makeTestRecorder(mLegacyStatsDir, PREFIX_DEV, + mSettings.getDevConfig(), includeTags); + return recorder.getOrLoadCompleteLocked(); + } + private void assertNetworkTotal(NetworkTemplate template, long rxBytes, long rxPackets, long txBytes, long txPackets, int operations) throws Exception { assertNetworkTotal(template, Long.MIN_VALUE, Long.MAX_VALUE, rxBytes, rxPackets, txBytes,