Merge changes I8bc68d10,Ie6182fa4 am: 8fa5ddae14

Original change: https://android-review.googlesource.com/c/platform/packages/modules/common/+/2046463

Change-Id: Ia46c9466fdd242f69bec23b39006e09d82eeaa3e
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

Android.bp

@@ -70,6 +70,17 @@ soong_config_module_type {
         "prefer",
     ],
 }
+
+soong_config_module_type {
+    name: "module_java_test_import",
+    module_type: "java_test_import",
+    config_namespace: "ANDROID",
+    bool_variables: ["module_build_from_source"],
+    properties: [
+        "prefer",
+    ],
+}
+
 soong_config_module_type {
     name: "module_java_sdk_library_import",
     module_type: "java_sdk_library_import",

build/mainline_modules_sdks.py

@@ -460,15 +460,27 @@ def create_dist_snapshot_for_r(build_release: BuildRelease,
 def create_sdk_snapshots_in_soong(build_release: BuildRelease,
                                   producer: "SdkDistProducer",
                                   modules: List["MainlineModule"]):
-    """Builds sdks and populates the dist."""
+    """Builds sdks and populates the dist for unbundled modules."""
-    producer.produce_dist_for_build_release(build_release, modules)
+    producer.produce_unbundled_dist_for_build_release(build_release, modules)
+
+
+def create_latest_sdk_snapshots(build_release: BuildRelease,
+                                producer: "SdkDistProducer",
+                                modules: List["MainlineModule"]):
+    """Builds and populates the latest release, including bundled modules."""
+    producer.produce_unbundled_dist_for_build_release(build_release, modules)
+    producer.produce_bundled_dist_for_build_release(build_release, modules)
 
 
 def create_legacy_dist_structures(build_release: BuildRelease,
                                   producer: "SdkDistProducer",
                                   modules: List["MainlineModule"]):
     """Creates legacy file structures."""
-    snapshots_dir = producer.produce_dist_for_build_release(
+
+    # Only put unbundled modules in the legacy dist and stubs structures.
+    modules = [m for m in modules if not m.is_bundled()]
+
+    snapshots_dir = producer.produce_unbundled_dist_for_build_release(
         build_release, modules)
 
     # Create the out/dist/mainline-sdks/stubs structure.
@@ -525,7 +537,7 @@ Tiramisu = BuildRelease(
 # before LEGACY_BUILD_RELEASE.
 LATEST = BuildRelease(
     name="latest",
-    creator=create_sdk_snapshots_in_soong,
+    creator=create_latest_sdk_snapshots,
     # There are no build release specific environment variables to pass to
     # Soong.
     soong_env={},
@@ -568,7 +580,11 @@ class ForRBuild:
 
 @dataclasses.dataclass(frozen=True)
 class MainlineModule:
-    """Represents a mainline module"""
+    """Represents an unbundled mainline module.
+
+    This is a module that is distributed as a prebuilt and intended to be
+    updated with Mainline trains.
+    """
     # The name of the apex.
     apex: str
 
@@ -603,6 +619,10 @@ class MainlineModule:
 
     for_r_build: typing.Optional[ForRBuild] = None
 
+    def is_bundled(self):
+        """Returns true for bundled modules. See BundledMainlineModule."""
+        return False
+
     def transformations(self, build_release):
         """Returns the transformations to apply to this module's snapshot(s)."""
         transformations = []
@@ -620,6 +640,23 @@ class MainlineModule:
         return self.first_release <= target_build_release
 
 
+@dataclasses.dataclass(frozen=True)
+class BundledMainlineModule(MainlineModule):
+    """Represents a bundled Mainline module or a platform SDK for module use.
+
+    A bundled module is always preloaded into the platform images.
+    """
+
+    def is_bundled(self):
+        return True
+
+    def transformations(self, build_release):
+        # Bundled modules are only used on thin branches where the corresponding
+        # sources are absent, so skip transformations and keep the default
+        # `prefer: false`.
+        return []
+
+
 # List of mainline modules.
 MAINLINE_MODULES = [
     MainlineModule(
@@ -727,6 +764,42 @@ MAINLINE_MODULES = [
     ),
 ]
 
+# List of Mainline modules that currently are never built unbundled. They should
+# not specify first_release, and they don't have com.google.android
+# counterparts.
+BUNDLED_MAINLINE_MODULES = [
+    BundledMainlineModule(
+        apex="com.android.i18n",
+        sdks=[
+            "i18n-module-sdk",
+            "i18n-module-test-exports",
+            "i18n-module-host-exports",
+        ],
+    ),
+    BundledMainlineModule(
+        apex="com.android.runtime",
+        sdks=[
+            "runtime-module-host-exports",
+            "runtime-module-sdk",
+        ],
+    ),
+    BundledMainlineModule(
+        apex="com.android.tzdata",
+        sdks=["tzdata-module-test-exports"],
+    ),
+]
+
+# List of platform SDKs for Mainline module use.
+PLATFORM_SDKS_FOR_MAINLINE = [
+    BundledMainlineModule(
+        apex="platform-mainline",
+        sdks=[
+            "platform-mainline-sdk",
+            "platform-mainline-test-exports",
+        ],
+    ),
+]
+
 
 @dataclasses.dataclass
 class SdkDistProducer:
@@ -751,17 +824,26 @@ class SdkDistProducer:
     # transformed to document where the changes came from.
     script: str = sys.argv[0]
 
-    # The path to the mainline-sdks dist directory.
+    # The path to the mainline-sdks dist directory for unbundled modules.
     #
     # Initialized in __post_init__().
     mainline_sdks_dir: str = dataclasses.field(init=False)
 
+    # The path to the mainline-sdks dist directory for bundled modules and
+    # platform SDKs.
+    #
+    # Initialized in __post_init__().
+    bundled_mainline_sdks_dir: str = dataclasses.field(init=False)
+
     def __post_init__(self):
         self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks")
+        self.bundled_mainline_sdks_dir = os.path.join(self.dist_dir,
+                                                      "bundled-mainline-sdks")
 
     def prepare(self):
-        # Clear the mainline-sdks dist directory.
+        # Clear the sdk dist directories.
         shutil.rmtree(self.mainline_sdks_dir, ignore_errors=True)
+        shutil.rmtree(self.bundled_mainline_sdks_dir, ignore_errors=True)
 
     def produce_dist(self, modules, build_releases):
         # Prepare the dist directory for the sdks.
@@ -788,38 +870,60 @@ class SdkDistProducer:
 
         snapshot_dir = self.snapshot_builder.build_snapshots_for_build_r(
             build_release, sdk_versions, modules)
-        self.populate_dist(build_release, sdk_versions, modules, snapshot_dir)
+        self.populate_unbundled_dist(build_release, sdk_versions, modules,
+                                     snapshot_dir)
 
-    def produce_dist_for_build_release(self, build_release, modules):
+    def produce_unbundled_dist_for_build_release(self, build_release, modules):
+        modules = [m for m in modules if not m.is_bundled()]
         sdk_versions = build_release.sdk_versions
         snapshots_dir = self.snapshot_builder.build_snapshots(
             build_release, sdk_versions, modules)
-        self.populate_dist(build_release, sdk_versions, modules, snapshots_dir)
+        self.populate_unbundled_dist(build_release, sdk_versions, modules,
+                                     snapshots_dir)
         return snapshots_dir
 
-    def populate_dist(self, build_release, sdk_versions, modules,
+    def produce_bundled_dist_for_build_release(self, build_release, modules):
+        modules = [m for m in modules if m.is_bundled()]
+        sdk_versions = build_release.sdk_versions
+        snapshots_dir = self.snapshot_builder.build_snapshots(
+            build_release, sdk_versions, modules)
+        self.populate_bundled_dist(build_release, modules, snapshots_dir)
+        return snapshots_dir
+
+    def populate_unbundled_dist(self, build_release, sdk_versions, modules,
                                 snapshots_dir):
         build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
                                               build_release.sub_dir)
-
         for module in modules:
-            apex = module.apex
             for sdk_version in sdk_versions:
                 for sdk in module.sdks:
-                    subdir = re.sub("^[^-]+-(module-)?", "", sdk)
+                    sdk_dist_dir = os.path.join(
+                        build_release_dist_dir, sdk_version)
+                    self.populate_dist_snapshot(
+                        build_release, module, sdk, sdk_dist_dir, sdk_version,
+                        snapshots_dir)
+
+    def populate_bundled_dist(self, build_release, modules, snapshots_dir):
+        sdk_dist_dir = self.bundled_mainline_sdks_dir
+        for module in modules:
+            for sdk in module.sdks:
+                self.populate_dist_snapshot(
+                    build_release, module, sdk, sdk_dist_dir, "current",
+                    snapshots_dir)
+
+    def populate_dist_snapshot(self, build_release, module, sdk, sdk_dist_dir,
+                               sdk_version, snapshots_dir):
+        subdir = re.sub("^.+-(sdk|(host|test)-exports)$", r'\1', sdk)
         if subdir not in ("sdk", "host-exports", "test-exports"):
             raise Exception(
-                            f"{sdk} is not a valid name, expected name in the"
+                f"{sdk} is not a valid name, expected it to end"
-                            f" format of"
+                f" with -(sdk|host-exports|test-exports)"
-                            f" ^[^-]+-(module-)?(sdk|host-exports|test-exports)"
             )
 
-                    sdk_dist_dir = os.path.join(build_release_dist_dir,
+        sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, subdir)
-                                                sdk_version, apex, subdir)
+        sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk, sdk_version)
-                    sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk,
-                                                     sdk_version)
         transformations = module.transformations(build_release)
-                    self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_dir,
+        self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_subdir,
                                    transformations)
 
     def dist_sdk_snapshot_zip(self, src_sdk_zip, sdk_dist_dir, transformations):
@@ -956,12 +1060,10 @@ def google_to_aosp_name(name):
     return name.replace("com.google.android.", "com.android.")
 
 
-def filter_modules(modules):
+def filter_modules(modules, target_build_apps):
-    target_build_apps = os.environ.get("TARGET_BUILD_APPS")
     if target_build_apps:
         target_build_apps = target_build_apps.split()
         return [m for m in modules if m.apex in target_build_apps]
-
     return modules
 
 
@@ -981,7 +1083,16 @@ def main(args):
         "--build-release",
         action="append",
         choices=[br.name for br in ALL_BUILD_RELEASES],
-        help="A target build for which snapshots are required.",
+        help="A target build for which snapshots are required. "
+        "If it is \"latest\" then Mainline module SDKs from platform and "
+        "bundled modules are included.",
+    )
+    args_parser.add_argument(
+        "--build-platform-sdks-for-mainline",
+        action="store_true",
+        help="Also build the platform SDKs for Mainline modules. "
+        "Defaults to true when TARGET_BUILD_APPS is not set. "
+        "Applicable only if the \"latest\" build release is built.",
     )
     args = args_parser.parse_args(args)
 
@@ -993,9 +1104,16 @@ def main(args):
             if b.name.lower() in selected_build_releases
         ]
 
-    producer = create_producer(args.tool_path)
+    target_build_apps = os.environ.get("TARGET_BUILD_APPS")
-    modules = filter_modules(MAINLINE_MODULES)
+    modules = filter_modules(MAINLINE_MODULES + BUNDLED_MAINLINE_MODULES,
+                             target_build_apps)
 
+    # Also build the platform Mainline SDKs either if no specific modules are
+    # requested or if --build-platform-sdks-for-mainline is given.
+    if not target_build_apps or args.build_platform_sdks_for_mainline:
+        modules += PLATFORM_SDKS_FOR_MAINLINE
+
+    producer = create_producer(args.tool_path)
     producer.produce_dist(modules, build_releases)
 
 

build/mainline_modules_sdks_test.py

@@ -17,6 +17,7 @@
 import re
 from pathlib import Path
 import os
+import shutil
 import tempfile
 import unittest
 import zipfile
@@ -24,7 +25,10 @@ from unittest import mock
 
 import mainline_modules_sdks as mm
 
-MAINLINE_MODULES_BY_APEX = dict((m.apex, m) for m in mm.MAINLINE_MODULES)
+MAINLINE_MODULES_BY_APEX = dict((m.apex, m) for m in (
+    mm.MAINLINE_MODULES +
+    mm.BUNDLED_MAINLINE_MODULES +
+    mm.PLATFORM_SDKS_FOR_MAINLINE))
 
 
 class FakeSnapshotBuilder(mm.SnapshotBuilder):
@@ -67,29 +71,47 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
 
 class TestProduceDist(unittest.TestCase):
 
-    def test(self):
+    def setUp(self):
-        """Verify the dist/mainline-sdks directory is populated correctly"""
+        self.tmp_dir = tempfile.mkdtemp()
-        with tempfile.TemporaryDirectory() as tmp_dir:
+        self.tmp_out_dir = os.path.join(self.tmp_dir, "out")
-            tmp_out_dir = os.path.join(tmp_dir, "out")
+        os.mkdir(self.tmp_out_dir)
-            os.mkdir(tmp_out_dir)
+        self.tmp_dist_dir = os.path.join(self.tmp_dir, "dist")
-            tmp_dist_dir = os.path.join(tmp_dir, "dist")
+        os.mkdir(self.tmp_dist_dir)
-            os.mkdir(tmp_dist_dir)
 
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def produce_dist(self, modules, build_releases):
+        subprocess_runner = mm.SubprocessRunner()
+        snapshot_builder = FakeSnapshotBuilder(
+            tool_path="path/to/mainline_modules_sdks.sh",
+            subprocess_runner=subprocess_runner,
+            out_dir=self.tmp_out_dir,
+        )
+        producer = mm.SdkDistProducer(
+            subprocess_runner=subprocess_runner,
+            snapshot_builder=snapshot_builder,
+            dist_dir=self.tmp_dist_dir,
+        )
+        producer.produce_dist(modules, build_releases)
+
+    def list_files_in_dir(self, tmp_dist_dir):
+        files = []
+        for abs_dir, _, filenames in os.walk(tmp_dist_dir):
+            rel_dir = os.path.relpath(abs_dir, tmp_dist_dir)
+            if rel_dir == ".":
+                rel_dir = ""
+            for f in filenames:
+                files.append(os.path.join(rel_dir, f))
+        return files
+
+    def test_unbundled_modules(self):
         modules = [
             MAINLINE_MODULES_BY_APEX["com.android.art"],
             MAINLINE_MODULES_BY_APEX["com.android.ipsec"],
             # Create a google specific module.
             mm.aosp_to_google(MAINLINE_MODULES_BY_APEX["com.android.wifi"]),
         ]
-
-            subprocess_runner = mm.SubprocessRunner()
-
-            snapshot_builder = FakeSnapshotBuilder(
-                tool_path="path/to/mainline_modules_sdks.sh",
-                subprocess_runner=subprocess_runner,
-                out_dir=tmp_out_dir,
-            )
-
         build_releases = [
             mm.Q,
             mm.R,
@@ -97,14 +119,7 @@ class TestProduceDist(unittest.TestCase):
             mm.LATEST,
             mm.LEGACY_BUILD_RELEASE,
         ]
-
+        self.produce_dist(modules, build_releases)
-            producer = mm.SdkDistProducer(
-                subprocess_runner=subprocess_runner,
-                snapshot_builder=snapshot_builder,
-                dist_dir=tmp_dist_dir,
-            )
-
-            producer.produce_dist(modules, build_releases)
 
         # pylint: disable=line-too-long
         self.assertEqual(
@@ -142,10 +157,10 @@ class TestProduceDist(unittest.TestCase):
                 "stubs/com.google.android.wifi/sdk_library/public/framework-wifi.srcjar",
                 "stubs/com.google.android.wifi/sdk_library/public/framework-wifi.txt",
             ],
-                sorted(self.list_files_in_dir(tmp_dist_dir)))
+            sorted(self.list_files_in_dir(self.tmp_dist_dir)))
 
         r_snaphot_dir = os.path.join(
-                tmp_out_dir, "soong/mainline-sdks/test/for-R-build")
+            self.tmp_out_dir, "soong/mainline-sdks/test/for-R-build")
         aosp_ipsec_r_bp_file = "com.android.ipsec/Android.bp"
         google_wifi_android_bp = "com.google.android.wifi/Android.bp"
         self.assertEqual([
@@ -178,15 +193,71 @@ class TestProduceDist(unittest.TestCase):
         expected = read_test_data("google_wifi_for_r_Android.bp")
         self.assertEqual(expected, wifi_contents)
 
-    def list_files_in_dir(self, tmp_dist_dir):
+    def test_old_release(self):
-        files = []
+        modules = [
-        for abs_dir, _, filenames in os.walk(tmp_dist_dir):
+            MAINLINE_MODULES_BY_APEX["com.android.art"], # An unnbundled module
-            rel_dir = os.path.relpath(abs_dir, tmp_dist_dir)
+            MAINLINE_MODULES_BY_APEX["com.android.runtime"], # A bundled module
-            if rel_dir == ".":
+            MAINLINE_MODULES_BY_APEX["platform-mainline"], # Platform SDK
-                rel_dir = ""
+        ]
-            for f in filenames:
+        build_releases = [mm.S]
-                files.append(os.path.join(rel_dir, f))
+        self.produce_dist(modules, build_releases)
-        return files
+
+        # pylint: disable=line-too-long
+        self.assertEqual(
+            [
+                "mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip",
+                "mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+            ],
+            sorted(self.list_files_in_dir(self.tmp_dist_dir)))
+
+    def test_latest_release(self):
+        modules = [
+            MAINLINE_MODULES_BY_APEX["com.android.art"], # An unnbundled module
+            MAINLINE_MODULES_BY_APEX["com.android.runtime"], # A bundled module
+            MAINLINE_MODULES_BY_APEX["platform-mainline"], # Platform SDK
+        ]
+        build_releases = [mm.LATEST]
+        self.produce_dist(modules, build_releases)
+
+        # pylint: disable=line-too-long
+        self.assertEqual(
+            [
+                # Bundled modules and platform SDKs.
+                "bundled-mainline-sdks/com.android.runtime/host-exports/runtime-module-host-exports-current.zip",
+                "bundled-mainline-sdks/com.android.runtime/sdk/runtime-module-sdk-current.zip",
+                "bundled-mainline-sdks/platform-mainline/sdk/platform-mainline-sdk-current.zip",
+                "bundled-mainline-sdks/platform-mainline/test-exports/platform-mainline-test-exports-current.zip",
+                # Unbundled (normal) modules.
+                "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+            ],
+            sorted(self.list_files_in_dir(self.tmp_dist_dir)))
+
+    def test_legacy_release(self):
+        modules = [
+            MAINLINE_MODULES_BY_APEX["com.android.art"], # An unnbundled module
+            MAINLINE_MODULES_BY_APEX["com.android.runtime"], # A bundled module
+            MAINLINE_MODULES_BY_APEX["platform-mainline"], # Platform SDK
+        ]
+        build_releases = [mm.LEGACY_BUILD_RELEASE]
+        self.produce_dist(modules, build_releases)
+
+        # pylint: disable=line-too-long
+        self.assertEqual(
+            [
+                # Legacy copy of the snapshots.
+                "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip",
+                "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+                # Legacy stubs directory containing unpacked java_sdk_library artifacts.
+                "stubs/com.android.art/sdk_library/public/art-removed.txt",
+                "stubs/com.android.art/sdk_library/public/art-stubs.jar",
+                "stubs/com.android.art/sdk_library/public/art.srcjar",
+                "stubs/com.android.art/sdk_library/public/art.txt",
+            ],
+            sorted(self.list_files_in_dir(self.tmp_dist_dir)))
 
 
 def path_to_test_data(relative_path):
@@ -290,15 +361,30 @@ class TestSoongConfigBoilerplateInserter(unittest.TestCase):
 class TestFilterModules(unittest.TestCase):
 
     def test_no_filter(self):
-        modules = mm.filter_modules(mm.MAINLINE_MODULES)
+        all_modules = mm.MAINLINE_MODULES + mm.BUNDLED_MAINLINE_MODULES
-        self.assertEqual(modules, mm.MAINLINE_MODULES)
+        modules = mm.filter_modules(all_modules, None)
+        self.assertEqual(modules, all_modules)
 
     def test_with_filter(self):
-        os.environ["TARGET_BUILD_APPS"] = "com.android.art"
+        modules = mm.filter_modules(mm.MAINLINE_MODULES, "com.android.art")
-        modules = mm.filter_modules(mm.MAINLINE_MODULES)
         expected = MAINLINE_MODULES_BY_APEX["com.android.art"]
         self.assertEqual(modules, [expected])
 
 
+class TestModuleProperties(unittest.TestCase):
+
+    def test_unbundled(self):
+        for module in mm.MAINLINE_MODULES:
+            with self.subTest(module=module):
+                self.assertFalse(module.is_bundled())
+
+    def test_bundled(self):
+        for module in (mm.BUNDLED_MAINLINE_MODULES +
+                       mm.PLATFORM_SDKS_FOR_MAINLINE):
+            with self.subTest(module=module):
+                self.assertTrue(module.is_bundled())
+                self.assertEqual(module.first_release, mm.LATEST)
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)