LineageOS/android_packages_modules_Connectivity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Skip presubmit for flaky tests

Browse Source 
Skip presubmit for tests that have been reported to be >0.5% flaky in
presubmit, or in mainline-presubmit. Tests that only meet that criteria
in mainline-presubmit are only marked as @SkipMainlinePresubmit.

Ensure that the tests are still run in postsubmit so they can be moved
back to presubmit after getting fixed.

Test: TH
Bug: 284534251
Bug: 292174275
Bug: 292173706
Bug: 292174832
Bug: 289879947
Bug: 286701510
Bug: 293239598
Bug: 279846955
Bug: 277567450
Bug: 293239828
Bug: 293240365
Change-Id: I986a1079709057d3210016c50b632fe0f3bff1e2
This commit is contained in:
Remi NGUYEN VAN
2023-07-26 16:15:03 +09:00
parent b8fc755259
commit 8db52a5987
5 changed files with 54 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
TEST_MAPPING
View File

@@ -126,6 +126,15 @@
{
"name": "FrameworksNetDeflakeTest"
},
// Postsubmit on virtual devices to monitor flakiness of @SkipPresubmit methods
{
"name": "CtsNetTestCases",
"options": [
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
},
{
"name": "FrameworksNetTests"
}
@@ -137,6 +146,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
@@ -148,6 +160,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
@@ -159,6 +174,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
@@ -170,6 +188,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
@@ -183,6 +204,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
},
@@ -204,6 +228,9 @@
{
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
"exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
},
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
},
@@ -262,6 +289,15 @@
"exclude-annotation": "com.android.testutils.NetworkStackModuleTest"
}
]
},
// Postsubmit on virtual devices to monitor flakiness of @SkipMainlinePresubmit methods
{
"name": "CtsNetTestCasesLatestSdk[CaptivePortalLoginGoogle.apk+NetworkStackGoogle.apk+com.google.android.resolv.apex+com.google.android.tethering.apex]",
"options": [
{
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
}
],
"imports": [

2
tests/cts/net/src/android/net/cts/CaptivePortalTest.kt
View File

@@ -49,6 +49,7 @@ import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTT
import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTTP_URL
import com.android.testutils.DeviceConfigRule
import com.android.testutils.RecorderCallback.CallbackEntry.CapabilitiesChanged
import com.android.testutils.SkipMainlinePresubmit
import com.android.testutils.TestHttpServer
import com.android.testutils.TestHttpServer.Request
import com.android.testutils.TestableNetworkCallback
@@ -137,6 +138,7 @@ class CaptivePortalTest {
}
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
fun testCaptivePortalIsNotDefaultNetwork() {
assumeTrue(pm.hasSystemFeature(FEATURE_TELEPHONY))
assumeTrue(pm.hasSystemFeature(FEATURE_WIFI))

9
tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java
View File

@@ -197,6 +197,8 @@ import com.android.testutils.DeviceConfigRule;
import com.android.testutils.DeviceInfoUtils;
import com.android.testutils.DumpTestUtils;
import com.android.testutils.RecorderCallback.CallbackEntry;
import com.android.testutils.SkipMainlinePresubmit;
import com.android.testutils.SkipPresubmit;
import com.android.testutils.TestHttpServer;
import com.android.testutils.TestNetworkTracker;
import com.android.testutils.TestableNetworkCallback;
@@ -1017,6 +1019,7 @@ public class ConnectivityManagerTest {
@AppModeFull(reason = "WRITE_SECURE_SETTINGS permission can't be granted to instant apps")
@Test @IgnoreUpTo(Build.VERSION_CODES.Q)
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testIsPrivateDnsBroken() throws InterruptedException {
final String invalidPrivateDnsServer = "invalidhostname.example.com";
final String goodPrivateDnsServer = "dns.google";
@@ -1124,6 +1127,7 @@ public class ConnectivityManagerTest {
*/
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRegisterNetworkCallback_withPendingIntent() {
assumeTrue(mPackageManager.hasSystemFeature(FEATURE_WIFI));
@@ -1267,6 +1271,7 @@ public class ConnectivityManagerTest {
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRegisterNetworkRequest_identicalPendingIntents() throws Exception {
runIdenticalPendingIntentsRequestTest(false /* useListen */);
}
@@ -2129,6 +2134,7 @@ public class ConnectivityManagerTest {
*/
@AppModeFull(reason = "NETWORK_AIRPLANE_MODE permission can't be granted to instant apps")
@Test
@SkipPresubmit(reason = "Out of SLO flakiness")
public void testSetAirplaneMode() throws Exception{
// Starting from T, wifi supports airplane mode enhancement which may not disconnect wifi
// when airplane mode is on. The actual behavior that the device will have could only be
@@ -2738,6 +2744,7 @@ public class ConnectivityManagerTest {
*/
@AppModeFull(reason = "Instant apps cannot create test networks")
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testSetOemNetworkPreferenceForTestOnlyPref() throws Exception {
// Cannot use @IgnoreUpTo(Build.VERSION_CODES.R) because this test also requires API 31
// shims, and @IgnoreUpTo does not check that.
@@ -2892,6 +2899,7 @@ public class ConnectivityManagerTest {
@AppModeFull(reason = "WRITE_DEVICE_CONFIG permission can't be granted to instant apps")
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRejectPartialConnectivity_TearDownNetwork() throws Exception {
assumeTrue(TestUtils.shouldTestSApis());
assumeTrue("testAcceptPartialConnectivity_validatedNetwork cannot execute"
@@ -3197,6 +3205,7 @@ public class ConnectivityManagerTest {
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
@SkipPresubmit(reason = "Out of SLO flakiness")
public void testMobileDataPreferredUids() throws Exception {
assumeTrue(TestUtils.shouldTestSApis());
final boolean canRunTest = mPackageManager.hasSystemFeature(FEATURE_WIFI)

5
tests/cts/net/src/android/net/cts/Ikev2VpnTest.java
View File

@@ -71,6 +71,8 @@ import com.android.testutils.DevSdkIgnoreRule;
import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
import com.android.testutils.DevSdkIgnoreRunner;
import com.android.testutils.RecorderCallback.CallbackEntry;
import com.android.testutils.SkipMainlinePresubmit;
import com.android.testutils.SkipPresubmit;
import com.android.testutils.TestableNetworkCallback;
import org.bouncycastle.x509.X509V1CertificateGenerator;
@@ -640,6 +642,7 @@ public class Ikev2VpnTest {
}
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV4() throws Exception {
doTestStartStopVpnProfile(false /* testIpv6Only */, false /* requiresValidation */,
false /* testSessionKey */, false /* testIkeTunConnParams */);
@@ -653,12 +656,14 @@ public class Ikev2VpnTest {
}
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV6() throws Exception {
doTestStartStopVpnProfile(true /* testIpv6Only */, false /* requiresValidation */,
false /* testSessionKey */, false /* testIkeTunConnParams */);
}
@Test @IgnoreUpTo(SC_V2)
@SkipPresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV6WithValidation() throws Exception {
assumeTrue(TestUtils.shouldTestTApis());
doTestStartStopVpnProfile(true /* testIpv6Only */, true /* requiresValidation */,

2
tests/cts/net/src/android/net/cts/IpSecManagerTest.java
View File

@@ -81,6 +81,7 @@ import androidx.test.runner.AndroidJUnit4;
import com.android.modules.utils.build.SdkLevel;
import com.android.testutils.DevSdkIgnoreRule;
import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
import com.android.testutils.SkipMainlinePresubmit;
import org.junit.Rule;
import org.junit.Test;
@@ -716,6 +717,7 @@ public class IpSecManagerTest extends IpSecBaseTest {
}
@Test
@SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testIkeOverUdpEncapSocket() throws Exception {
// IPv6 not supported for UDP-encap-ESP
InetAddress local = InetAddress.getByName(IPV4_LOOPBACK);