Merge "Support building build target specific snapshots" am: 27aef8152b

Original change: https://android-review.googlesource.com/c/platform/packages/modules/common/+/1958805

Change-Id: Ic6d998f91943e25964b3e2868dfc304a2f80089b

build/mainline_modules_sdks.py

@@ -27,6 +27,8 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import typing
+from typing import Callable, List
 import zipfile
 
 
@@ -214,7 +216,7 @@ class SnapshotBuilder:
         return os.path.join(self.get_mainline_sdks_path(),
                             f"{sdk_name}-{sdk_version}.zip")
 
-    def build_snapshots(self, sdk_versions, modules):
+    def build_snapshots(self, build_release, sdk_versions, modules):
         # Build the SDKs once for each version.
         for sdk_version in sdk_versions:
             # Compute the paths to all the Soong generated sdk snapshot files
@@ -225,16 +227,20 @@ class SnapshotBuilder:
                 for sdk in module.sdks
             ]
 
-            # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but we
+            # Extra environment variables to pass to the build process.
-            # currently break without it.
-            #
-            # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside sdk
-            # zip files as expected by prebuilt drop.
             extraEnv = {
+                # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but
+                #  we currently break without it.
                 "SOONG_ALLOW_MISSING_DEPENDENCIES": "true",
+                # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside
+                # sdk zip files as expected by prebuilt drop.
                 "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true",
+                # Set SOONG_SDK_SNAPSHOT_VERSION to generate the appropriately
+                # tagged version of the sdk.
                 "SOONG_SDK_SNAPSHOT_VERSION": sdk_version,
             }
+            extraEnv.update(build_release.soong_env)
+
             # Unless explicitly specified in the calling environment set
             # TARGET_BUILD_VARIANT=user.
             # This MUST be identical to the TARGET_BUILD_VARIANT used to build
@@ -256,6 +262,149 @@ class SnapshotBuilder:
             self.subprocess_runner.run(cmd, env=env)
 
 
+# A list of the sdk versions to build. Usually just current but can include a
+# numeric version too.
+SDK_VERSIONS = [
+    # Suitable for overriding the source modules with prefer:true.
+    # Unlike "unversioned" this mode also adds "@current" suffixed modules
+    # with the same prebuilts (which are never preferred).
+    "current",
+    # Insert additional sdk versions needed for the latest build release.
+]
+
+# The initially empty list of build releases. Every BuildRelease that is created
+# automatically appends itself to this list.
+ALL_BUILD_RELEASES = []
+
+
+@dataclasses.dataclass(frozen=True)
+class BuildRelease:
+    """Represents a build release"""
+
+    # The name of the build release, e.g. Q, R, S, T, etc.
+    name: str
+
+    # The function to call to create the snapshot in the dist, that covers
+    # building and copying the snapshot into the dist.
+    creator: Callable[
+        ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None]
+
+    # The sub-directory of dist/mainline-sdks into which the build release
+    # specific snapshots will be copied.
+    #
+    # Defaults to for-<name>-build.
+    sub_dir: str = None
+
+    # Additional environment variables to pass to Soong when building the
+    # snapshots for this build release.
+    #
+    # Defaults to {
+    #     "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>,
+    # }
+    soong_env: typing.Dict[str, str] = None
+
+    # The sdk versions that need to be generated for this build release.
+    sdk_versions: List[str] = \
+        dataclasses.field(default_factory=lambda: SDK_VERSIONS)
+
+    # The position of this instance within the BUILD_RELEASES list.
+    ordinal: int = dataclasses.field(default=-1, init=False)
+
+    def __post_init__(self):
+        # The following use object.__setattr__ as this object is frozen and
+        # attempting to set the fields directly would cause an exception to be
+        # thrown.
+        object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES))
+        # Add this to the end of the list of all build releases.
+        ALL_BUILD_RELEASES.append(self)
+        # If no sub_dir was specified then set the default.
+        if self.sub_dir is None:
+            object.__setattr__(self, "sub_dir", f"for-{self.name}-build")
+        # If no soong_env was specified then set the default.
+        if self.soong_env is None:
+            object.__setattr__(
+                self,
+                "soong_env",
+                {
+                    # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a
+                    # snapshot suitable for a specific target build release.
+                    "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name,
+                })
+
+    def __le__(self, other):
+        return self.ordinal <= other.ordinal
+
+
+def create_no_dist_snapshot(build_release: BuildRelease,
+                            producer: "SdkDistProducer",
+                            modules: List["MainlineModule"]):
+    """A place holder dist snapshot creation function that does nothing."""
+    print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}")
+    return
+
+
+def create_sdk_snapshots_in_Soong(build_release: BuildRelease,
+                                  producer: "SdkDistProducer",
+                                  modules: List["MainlineModule"]):
+    """Builds sdks and populates the dist."""
+    producer.produce_dist_for_build_release(build_release, modules)
+    return
+
+
+def reuse_latest_sdk_snapshots(build_release: BuildRelease,
+                               producer: "SdkDistProducer",
+                               modules: List["MainlineModule"]):
+    """Copies the snapshots from the latest build."""
+    producer.populate_dist(build_release, build_release.sdk_versions, modules)
+    return
+
+
+Q = BuildRelease(
+    name="Q",
+    # At the moment we do not generate a snapshot for Q.
+    creator=create_no_dist_snapshot,
+)
+R = BuildRelease(
+    name="R",
+    # At the moment we do not generate a snapshot for R.
+    creator=create_no_dist_snapshot,
+)
+S = BuildRelease(
+    name="S",
+    # Generate a snapshot for S using Soong.
+    creator=create_sdk_snapshots_in_Soong,
+)
+
+# Insert additional BuildRelease definitions for following releases here,
+# before LATEST.
+
+# The build release for the latest build supported by this build, i.e. the
+# current build. This must be the last BuildRelease defined in this script,
+# before LEGACY_BUILD_RELEASE.
+LATEST = BuildRelease(
+    name="latest",
+    creator=create_sdk_snapshots_in_Soong,
+    # There are no build release specific environment variables to pass to
+    # Soong.
+    soong_env={},
+)
+
+# The build release to populate the legacy dist structure that does not specify
+# a particular build release. This MUST come after LATEST so that it includes
+# all the modules for which sdk snapshot source is available.
+LEGACY_BUILD_RELEASE = BuildRelease(
+    name="legacy",
+    # There is no build release specific sub directory.
+    sub_dir="",
+    # There are no build release specific environment variables to pass to
+    # Soong.
+    soong_env={},
+    # Do not create new snapshots, simply use the snapshots generated for
+    # latest.
+    creator=reuse_latest_sdk_snapshots,
+)
+
+
 @dataclasses.dataclass(frozen=True)
 class MainlineModule:
     """Represents a mainline module"""
@@ -265,6 +414,19 @@ class MainlineModule:
     # The names of the sdk and module_exports.
     sdks: list[str]
 
+    # The first build release in which the SDK snapshot for this module is
+    # needed.
+    #
+    # Note: This is not necessarily the same build release in which the SDK
+    #       source was first included. So, a module that was added in build T
+    #       could potentially be used in an S release and so its SDK will need
+    #       to be made available for S builds.
+    #
+    # Defaults to the latest build, i.e. the build on which this script is run
+    # as the snapshot is assumed to be needed in the build containing the sdk
+    # source.
+    first_release: BuildRelease = LATEST
+
     # The configuration variable, defaults to ANDROID:module_build_from_source
     configVar: ConfigVar = ConfigVar(
         namespace="ANDROID",
@@ -288,6 +450,10 @@ class MainlineModule:
                 configBpDefFile=self.configBpDefFile),
         ]
 
+    def is_required_for(self, target_build_release):
+        """True if this module is required for the target build release."""
+        return self.first_release <= target_build_release
+
 
 # List of mainline modules.
 MAINLINE_MODULES = [
@@ -298,6 +464,7 @@ MAINLINE_MODULES = [
             "art-module-test-exports",
             "art-module-host-exports",
         ],
+        first_release=S,
         # Override the config... fields.
         configVar=ConfigVar(
             namespace="art_module",
@@ -313,22 +480,27 @@ MAINLINE_MODULES = [
             "conscrypt-module-test-exports",
             "conscrypt-module-host-exports",
         ],
+        first_release=Q,
     ),
     MainlineModule(
         apex="com.android.ipsec",
         sdks=["ipsec-module-sdk"],
+        first_release=S,
     ),
     MainlineModule(
         apex="com.android.media",
         sdks=["media-module-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.mediaprovider",
         sdks=["mediaprovider-module-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.permission",
         sdks=["permission-module-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.scheduling",
@@ -337,30 +509,25 @@ MAINLINE_MODULES = [
     MainlineModule(
         apex="com.android.sdkext",
         sdks=["sdkextensions-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.os.statsd",
         sdks=["statsd-module-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.tethering",
         sdks=["tethering-module-sdk"],
+        first_release=R,
     ),
     MainlineModule(
         apex="com.android.wifi",
         sdks=["wifi-module-sdk"],
+        first_release=R,
     ),
 ]
 
-# A list of the sdk versions to build. Usually just current but can include a
-# numeric version too.
-SDK_VERSIONS = [
-    # Suitable for overriding the source modules with prefer:true.
-    # Unlike "unversioned" this mode also adds "@current" suffixed modules
-    # with the same prebuilts (which are never preferred).
-    "current",
-]
-
 
 @dataclasses.dataclass
 class SdkDistProducer:
@@ -385,14 +552,39 @@ class SdkDistProducer:
     # transformed to document where the changes came from.
     script: str = sys.argv[0]
 
-    def produce_dist(self, modules):
+    # The path to the mainline-sdks dist directory.
-        sdk_versions = SDK_VERSIONS
+    #
-        self.build_sdks(sdk_versions, modules)
+    # Initialized in __post_init__().
-        self.populate_dist(sdk_versions, modules)
+    mainline_sdks_dir: str = dataclasses.field(init=False)
+
+    def __post_init__(self):
+        self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks")
+
+    def prepare(self):
+        # Clear the mainline-sdks dist directory.
+        shutil.rmtree(self.mainline_sdks_dir, ignore_errors=True)
+
+    def produce_dist(self, modules, build_releases):
+        # Prepare the dist directory for the sdks.
+        self.prepare()
+
+        for build_release in build_releases:
+            # Only build modules that are required for this build release.
+            filtered_modules = [
+                m for m in modules if m.is_required_for(build_release)
+            ]
+            if filtered_modules:
+                print(f"Building SDK snapshots for {build_release.name}"
+                      f" build release")
+                build_release.creator(build_release, self, filtered_modules)
+
         self.populate_stubs(modules)
 
-    def build_sdks(self, sdk_versions, modules):
+    def produce_dist_for_build_release(self, build_release, modules):
-        self.snapshot_builder.build_snapshots(sdk_versions, modules)
+        sdk_versions = build_release.sdk_versions
+        self.snapshot_builder.build_snapshots(build_release, sdk_versions,
+                                              modules)
+        self.populate_dist(build_release, sdk_versions, modules)
 
     def unzip_current_stubs(self, sdk_name, apex_name):
         """Unzips stubs for "current" into {producer.dist_dir}/stubs/{apex}."""
@@ -418,10 +610,9 @@ class SdkDistProducer:
                 if sdk.endswith("-sdk"):
                     self.unzip_current_stubs(sdk, apex)
 
-    def populate_dist(self, sdk_versions, modules):
+    def populate_dist(self, build_release, sdk_versions, modules):
-        # Clear and populate the mainline-sdks dist directory.
+        build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
-        sdks_dist_dir = os.path.join(self.dist_dir, "mainline-sdks")
+                                              build_release.sub_dir)
-        shutil.rmtree(sdks_dist_dir, ignore_errors=True)
 
         for module in modules:
             apex = module.apex
@@ -435,8 +626,8 @@ class SdkDistProducer:
                             f" ^[^-]+-(module-)?(sdk|host-exports|test-exports)"
                         )
 
-                    sdk_dist_dir = os.path.join(sdks_dist_dir, sdk_version,
+                    sdk_dist_dir = os.path.join(build_release_dist_dir,
-                                                apex, subdir)
+                                                sdk_version, apex, subdir)
                     sdk_path = self.snapshot_builder.get_sdk_path(
                         sdk, sdk_version)
                     self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_dir,
@@ -563,7 +754,8 @@ def main():
 
     producer = create_producer()
     modules = filter_modules(MAINLINE_MODULES)
-    producer.produce_dist(modules)
+
+    producer.produce_dist(modules, ALL_BUILD_RELEASES)
 
 
 if __name__ == "__main__":

build/mainline_modules_sdks_test.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Unit tests for mainline_modules_sdks.py."""
+import dataclasses
 from pathlib import Path
 import os
 import tempfile
@@ -42,7 +43,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
                 z.writestr("sdk_library/public/lib.jar", "")
                 z.writestr("sdk_library/public/api.txt", "")
 
-    def build_snapshots(self, sdk_versions, modules):
+    def build_snapshots(self, build_release, sdk_versions, modules):
         # Create input file structure.
         sdks_out_dir = Path(self.get_mainline_sdks_path())
         sdks_out_dir.mkdir(parents=True, exist_ok=True)
@@ -75,13 +76,21 @@ class TestProduceDist(unittest.TestCase):
                 out_dir=tmp_out_dir,
             )
 
+            build_releases = [
+                mm.Q,
+                mm.R,
+                mm.S,
+                mm.LATEST,
+                mm.LEGACY_BUILD_RELEASE,
+            ]
+
             producer = mm.SdkDistProducer(
                 subprocess_runner=subprocess_runner,
                 snapshot_builder=snapshot_builder,
                 dist_dir=tmp_dist_dir,
             )
 
-            producer.produce_dist(modules)
+            producer.produce_dist(modules, build_releases)
 
             files = []
             for abs_dir, _, filenames in os.walk(tmp_dist_dir):
@@ -89,20 +98,33 @@ class TestProduceDist(unittest.TestCase):
                 for f in filenames:
                     files.append(os.path.join(rel_dir, f))
             # pylint: disable=line-too-long
-            self.assertEqual([
+            self.assertEqual(
-                "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                [
-                "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip",
+                    # Legacy copy of the snapshots, for use by tools that don't support build specific snapshots.
-                "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+                    "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip",
-                "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
+                    "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip",
-                "stubs/com.android.art/sdk_library/public/api.txt",
+                    "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip",
-                "stubs/com.android.art/sdk_library/public/lib.jar",
+                    "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
-                "stubs/com.android.art/sdk_library/public/removed.txt",
+                    # Build specific snapshots.
-                "stubs/com.android.art/sdk_library/public/source.srcjar",
+                    "mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
-                "stubs/com.android.ipsec/sdk_library/public/api.txt",
+                    "mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip",
-                "stubs/com.android.ipsec/sdk_library/public/lib.jar",
+                    "mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
-                "stubs/com.android.ipsec/sdk_library/public/removed.txt",
+                    "mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
-                "stubs/com.android.ipsec/sdk_library/public/source.srcjar",
+                    "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
-            ], sorted(files))
+                    "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
+                    "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+                    "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
+                    # Legacy stubs directory containing unpacked java_sdk_library artifacts.
+                    "stubs/com.android.art/sdk_library/public/api.txt",
+                    "stubs/com.android.art/sdk_library/public/lib.jar",
+                    "stubs/com.android.art/sdk_library/public/removed.txt",
+                    "stubs/com.android.art/sdk_library/public/source.srcjar",
+                    "stubs/com.android.ipsec/sdk_library/public/api.txt",
+                    "stubs/com.android.ipsec/sdk_library/public/lib.jar",
+                    "stubs/com.android.ipsec/sdk_library/public/removed.txt",
+                    "stubs/com.android.ipsec/sdk_library/public/source.srcjar",
+                ],
+                sorted(files))
 
 
 def pathToTestData(relative_path):