Skip presubmit for flaky tests

Skip presubmit for tests that have been reported to be >0.5% flaky in
presubmit, or in mainline-presubmit. Tests that only meet that criteria
in mainline-presubmit are only marked as @SkipMainlinePresubmit.

Ensure that the tests are still run in postsubmit so they can be moved
back to presubmit after getting fixed.

Test: TH
Bug: 284534251
Bug: 292174275
Bug: 292173706
Bug: 292174832
Bug: 289879947
Bug: 286701510
Bug: 293239598
Bug: 279846955
Bug: 277567450
Bug: 293239828
Bug: 293240365
Change-Id: I986a1079709057d3210016c50b632fe0f3bff1e2

TEST_MAPPING

@@ -126,6 +126,15 @@
     {
       "name": "FrameworksNetDeflakeTest"
     },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipPresubmit methods
+    {
+      "name": "CtsNetTestCases",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
+    },
     {
       "name": "FrameworksNetTests"
     }
@@ -137,6 +146,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -148,6 +160,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -159,6 +174,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -170,6 +188,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -183,6 +204,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
@@ -204,6 +228,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
@@ -262,6 +289,15 @@
           "exclude-annotation": "com.android.testutils.NetworkStackModuleTest"
         }
       ]
+    },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipMainlinePresubmit methods
+    {
+      "name": "CtsNetTestCasesLatestSdk[CaptivePortalLoginGoogle.apk+NetworkStackGoogle.apk+com.google.android.resolv.apex+com.google.android.tethering.apex]",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
     }
   ],
   "imports": [

tests/cts/net/src/android/net/cts/CaptivePortalTest.kt

@@ -49,6 +49,7 @@ import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTT
 import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTTP_URL
 import com.android.testutils.DeviceConfigRule
 import com.android.testutils.RecorderCallback.CallbackEntry.CapabilitiesChanged
+import com.android.testutils.SkipMainlinePresubmit
 import com.android.testutils.TestHttpServer
 import com.android.testutils.TestHttpServer.Request
 import com.android.testutils.TestableNetworkCallback
@@ -137,6 +138,7 @@ class CaptivePortalTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     fun testCaptivePortalIsNotDefaultNetwork() {
         assumeTrue(pm.hasSystemFeature(FEATURE_TELEPHONY))
         assumeTrue(pm.hasSystemFeature(FEATURE_WIFI))

tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java

@@ -197,6 +197,8 @@ import com.android.testutils.DeviceConfigRule;
 import com.android.testutils.DeviceInfoUtils;
 import com.android.testutils.DumpTestUtils;
 import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
 import com.android.testutils.TestHttpServer;
 import com.android.testutils.TestNetworkTracker;
 import com.android.testutils.TestableNetworkCallback;
@@ -1017,6 +1019,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "WRITE_SECURE_SETTINGS permission can't be granted to instant apps")
     @Test @IgnoreUpTo(Build.VERSION_CODES.Q)
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testIsPrivateDnsBroken() throws InterruptedException {
         final String invalidPrivateDnsServer = "invalidhostname.example.com";
         final String goodPrivateDnsServer = "dns.google";
@@ -1124,6 +1127,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRegisterNetworkCallback_withPendingIntent() {
         assumeTrue(mPackageManager.hasSystemFeature(FEATURE_WIFI));
 
@@ -1267,6 +1271,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRegisterNetworkRequest_identicalPendingIntents() throws Exception {
         runIdenticalPendingIntentsRequestTest(false /* useListen */);
     }
@@ -2129,6 +2134,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "NETWORK_AIRPLANE_MODE permission can't be granted to instant apps")
     @Test
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testSetAirplaneMode() throws Exception{
         // Starting from T, wifi supports airplane mode enhancement which may not disconnect wifi
         // when airplane mode is on. The actual behavior that the device will have could only be
@@ -2738,6 +2744,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "Instant apps cannot create test networks")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testSetOemNetworkPreferenceForTestOnlyPref() throws Exception {
         // Cannot use @IgnoreUpTo(Build.VERSION_CODES.R) because this test also requires API 31
         // shims, and @IgnoreUpTo does not check that.
@@ -2892,6 +2899,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "WRITE_DEVICE_CONFIG permission can't be granted to instant apps")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRejectPartialConnectivity_TearDownNetwork() throws Exception {
         assumeTrue(TestUtils.shouldTestSApis());
         assumeTrue("testAcceptPartialConnectivity_validatedNetwork cannot execute"
@@ -3197,6 +3205,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testMobileDataPreferredUids() throws Exception {
         assumeTrue(TestUtils.shouldTestSApis());
         final boolean canRunTest = mPackageManager.hasSystemFeature(FEATURE_WIFI)

tests/cts/net/src/android/net/cts/Ikev2VpnTest.java

@@ -71,6 +71,8 @@ import com.android.testutils.DevSdkIgnoreRule;
 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
 import com.android.testutils.DevSdkIgnoreRunner;
 import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
 import com.android.testutils.TestableNetworkCallback;
 
 import org.bouncycastle.x509.X509V1CertificateGenerator;
@@ -640,6 +642,7 @@ public class Ikev2VpnTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV4() throws Exception {
         doTestStartStopVpnProfile(false /* testIpv6Only */, false /* requiresValidation */,
                 false /* testSessionKey */, false /* testIkeTunConnParams */);
@@ -653,12 +656,14 @@ public class Ikev2VpnTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV6() throws Exception {
         doTestStartStopVpnProfile(true /* testIpv6Only */, false /* requiresValidation */,
                 false /* testSessionKey */, false /* testIkeTunConnParams */);
     }
 
     @Test @IgnoreUpTo(SC_V2)
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV6WithValidation() throws Exception {
         assumeTrue(TestUtils.shouldTestTApis());
         doTestStartStopVpnProfile(true /* testIpv6Only */, true /* requiresValidation */,

tests/cts/net/src/android/net/cts/IpSecManagerTest.java

@@ -81,6 +81,7 @@ import androidx.test.runner.AndroidJUnit4;
 import com.android.modules.utils.build.SdkLevel;
 import com.android.testutils.DevSdkIgnoreRule;
 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
+import com.android.testutils.SkipMainlinePresubmit;
 
 import org.junit.Rule;
 import org.junit.Test;
@@ -716,6 +717,7 @@ public class IpSecManagerTest extends IpSecBaseTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testIkeOverUdpEncapSocket() throws Exception {
         // IPv6 not supported for UDP-encap-ESP
         InetAddress local = InetAddress.getByName(IPV4_LOOPBACK);