Merge "Skip presubmit for flaky tests" into main

TEST_MAPPING

@@ -129,6 +129,15 @@
     {
       "name": "FrameworksNetDeflakeTest"
     },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipPresubmit methods
+    {
+      "name": "CtsNetTestCases",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
+    },
     {
       "name": "FrameworksNetTests"
     }
@@ -140,6 +149,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -151,6 +163,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -162,6 +177,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -173,6 +191,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
@@ -186,6 +207,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
@@ -207,6 +231,9 @@
         {
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
+        {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
         {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
@@ -268,6 +295,15 @@
           "exclude-annotation": "com.android.testutils.NetworkStackModuleTest"
         }
       ]
+    },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipMainlinePresubmit methods
+    {
+      "name": "CtsNetTestCasesLatestSdk[CaptivePortalLoginGoogle.apk+NetworkStackGoogle.apk+com.google.android.resolv.apex+com.google.android.tethering.apex]",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
     }
   ],
   "imports": [

tests/cts/net/src/android/net/cts/CaptivePortalTest.kt

@@ -49,6 +49,7 @@ import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTT
 import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTTP_URL
 import com.android.testutils.DeviceConfigRule
 import com.android.testutils.RecorderCallback.CallbackEntry.CapabilitiesChanged
+import com.android.testutils.SkipMainlinePresubmit
 import com.android.testutils.TestHttpServer
 import com.android.testutils.TestHttpServer.Request
 import com.android.testutils.TestableNetworkCallback
@@ -137,6 +138,7 @@ class CaptivePortalTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     fun testCaptivePortalIsNotDefaultNetwork() {
         assumeTrue(pm.hasSystemFeature(FEATURE_TELEPHONY))
         assumeTrue(pm.hasSystemFeature(FEATURE_WIFI))

tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java

@@ -197,6 +197,8 @@ import com.android.testutils.DeviceConfigRule;
 import com.android.testutils.DeviceInfoUtils;
 import com.android.testutils.DumpTestUtils;
 import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
 import com.android.testutils.TestHttpServer;
 import com.android.testutils.TestNetworkTracker;
 import com.android.testutils.TestableNetworkCallback;
@@ -1017,6 +1019,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "WRITE_SECURE_SETTINGS permission can't be granted to instant apps")
     @Test @IgnoreUpTo(Build.VERSION_CODES.Q)
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testIsPrivateDnsBroken() throws InterruptedException {
         final String invalidPrivateDnsServer = "invalidhostname.example.com";
         final String goodPrivateDnsServer = "dns.google";
@@ -1124,6 +1127,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRegisterNetworkCallback_withPendingIntent() {
         assumeTrue(mPackageManager.hasSystemFeature(FEATURE_WIFI));
 
@@ -1267,6 +1271,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRegisterNetworkRequest_identicalPendingIntents() throws Exception {
         runIdenticalPendingIntentsRequestTest(false /* useListen */);
     }
@@ -2129,6 +2134,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "NETWORK_AIRPLANE_MODE permission can't be granted to instant apps")
     @Test
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testSetAirplaneMode() throws Exception{
         // Starting from T, wifi supports airplane mode enhancement which may not disconnect wifi
         // when airplane mode is on. The actual behavior that the device will have could only be
@@ -2738,6 +2744,7 @@ public class ConnectivityManagerTest {
      */
     @AppModeFull(reason = "Instant apps cannot create test networks")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testSetOemNetworkPreferenceForTestOnlyPref() throws Exception {
         // Cannot use @IgnoreUpTo(Build.VERSION_CODES.R) because this test also requires API 31
         // shims, and @IgnoreUpTo does not check that.
@@ -2892,6 +2899,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "WRITE_DEVICE_CONFIG permission can't be granted to instant apps")
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testRejectPartialConnectivity_TearDownNetwork() throws Exception {
         assumeTrue(TestUtils.shouldTestSApis());
         assumeTrue("testAcceptPartialConnectivity_validatedNetwork cannot execute"
@@ -3197,6 +3205,7 @@ public class ConnectivityManagerTest {
 
     @AppModeFull(reason = "Cannot get WifiManager in instant app mode")
     @Test
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testMobileDataPreferredUids() throws Exception {
         assumeTrue(TestUtils.shouldTestSApis());
         final boolean canRunTest = mPackageManager.hasSystemFeature(FEATURE_WIFI)

tests/cts/net/src/android/net/cts/Ikev2VpnTest.java

@@ -71,6 +71,8 @@ import com.android.testutils.DevSdkIgnoreRule;
 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
 import com.android.testutils.DevSdkIgnoreRunner;
 import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
 import com.android.testutils.TestableNetworkCallback;
 
 import org.bouncycastle.x509.X509V1CertificateGenerator;
@@ -640,6 +642,7 @@ public class Ikev2VpnTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV4() throws Exception {
         doTestStartStopVpnProfile(false /* testIpv6Only */, false /* requiresValidation */,
                 false /* testSessionKey */, false /* testIkeTunConnParams */);
@@ -653,12 +656,14 @@ public class Ikev2VpnTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV6() throws Exception {
         doTestStartStopVpnProfile(true /* testIpv6Only */, false /* requiresValidation */,
                 false /* testSessionKey */, false /* testIkeTunConnParams */);
     }
 
     @Test @IgnoreUpTo(SC_V2)
+    @SkipPresubmit(reason = "Out of SLO flakiness")
     public void testStartStopVpnProfileV6WithValidation() throws Exception {
         assumeTrue(TestUtils.shouldTestTApis());
         doTestStartStopVpnProfile(true /* testIpv6Only */, true /* requiresValidation */,

tests/cts/net/src/android/net/cts/IpSecManagerTest.java

@@ -81,6 +81,7 @@ import androidx.test.runner.AndroidJUnit4;
 import com.android.modules.utils.build.SdkLevel;
 import com.android.testutils.DevSdkIgnoreRule;
 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
+import com.android.testutils.SkipMainlinePresubmit;
 
 import org.junit.Rule;
 import org.junit.Test;
@@ -716,6 +717,7 @@ public class IpSecManagerTest extends IpSecBaseTest {
     }
 
     @Test
+    @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
     public void testIkeOverUdpEncapSocket() throws Exception {
         // IPv6 not supported for UDP-encap-ESP
         InetAddress local = InetAddress.getByName(IPV4_LOOPBACK);