From 01ea57c593405963274c757bf24b1e95a8b4b6e7 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Mon, 17 Jan 2022 16:39:00 +0000 Subject: [PATCH] Support building build target specific snapshots This changes adds support for building target build release specific snapshots. It adds a BuildRelease class that defines the characteristics of the build release, e.g. name, how to create it, etc. It also adds a first_release field to MainlineModule which specifies the first BuildRelease for which a snapshot of the module is required and initializes that field for each module. After these changes this script will generate: 1. A legacy set of snapshots that match the file structure that was generated without this change. This is intended to allow existing consumers of the generated artifacts to continue to work while they are modified to make use of target build release specific snapshots. 2. The set of snapshots for the latest build release, i.e. the build release containing the source from which the snapshots are produced. This includes snapshots for all the modules. 3. For each build release from S onwards a set of snapshots for the modules it supports. It does not currently generate snapshots for Q and R releases as Soong cannot generate a compatible snapshot for them. If it is necessary to generate snapshots for those target build releases (similar to what the packages/modules/common/generate_ml_bundle.sh would produce) then a follow up change will add that capability to this script. Bug: 204763318 Test: atest mainline_modules_sdks_test packages/modules/common/build/mainline_modules_sdks.sh tree out/dist/mainline-sdks out/dist/stubs - check that it adds the for-latest-build directory but is otherwise identical to what was produced before this change. Change-Id: I48eb0b69cbe8664106b826ee0898557c98e039c2 --- build/mainline_modules_sdks.py | 248 ++++++++++++++++++++++++---- build/mainline_modules_sdks_test.py | 54 ++++-- 2 files changed, 258 insertions(+), 44 deletions(-) diff --git a/build/mainline_modules_sdks.py b/build/mainline_modules_sdks.py index 3fe0ddd..cddd3c9 100755 --- a/build/mainline_modules_sdks.py +++ b/build/mainline_modules_sdks.py @@ -27,6 +27,8 @@ import shutil import subprocess import sys import tempfile +import typing +from typing import Callable, List import zipfile @@ -214,7 +216,7 @@ class SnapshotBuilder: return os.path.join(self.get_mainline_sdks_path(), f"{sdk_name}-{sdk_version}.zip") - def build_snapshots(self, sdk_versions, modules): + def build_snapshots(self, build_release, sdk_versions, modules): # Build the SDKs once for each version. for sdk_version in sdk_versions: # Compute the paths to all the Soong generated sdk snapshot files @@ -225,16 +227,20 @@ class SnapshotBuilder: for sdk in module.sdks ] - # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but we - # currently break without it. - # - # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside sdk - # zip files as expected by prebuilt drop. + # Extra environment variables to pass to the build process. extraEnv = { + # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but + # we currently break without it. "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", + # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside + # sdk zip files as expected by prebuilt drop. "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", + # Set SOONG_SDK_SNAPSHOT_VERSION to generate the appropriately + # tagged version of the sdk. "SOONG_SDK_SNAPSHOT_VERSION": sdk_version, } + extraEnv.update(build_release.soong_env) + # Unless explicitly specified in the calling environment set # TARGET_BUILD_VARIANT=user. # This MUST be identical to the TARGET_BUILD_VARIANT used to build @@ -256,6 +262,149 @@ class SnapshotBuilder: self.subprocess_runner.run(cmd, env=env) +# A list of the sdk versions to build. Usually just current but can include a +# numeric version too. +SDK_VERSIONS = [ + # Suitable for overriding the source modules with prefer:true. + # Unlike "unversioned" this mode also adds "@current" suffixed modules + # with the same prebuilts (which are never preferred). + "current", + # Insert additional sdk versions needed for the latest build release. +] + +# The initially empty list of build releases. Every BuildRelease that is created +# automatically appends itself to this list. +ALL_BUILD_RELEASES = [] + + +@dataclasses.dataclass(frozen=True) +class BuildRelease: + """Represents a build release""" + + # The name of the build release, e.g. Q, R, S, T, etc. + name: str + + # The function to call to create the snapshot in the dist, that covers + # building and copying the snapshot into the dist. + creator: Callable[ + ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] + + # The sub-directory of dist/mainline-sdks into which the build release + # specific snapshots will be copied. + # + # Defaults to for--build. + sub_dir: str = None + + # Additional environment variables to pass to Soong when building the + # snapshots for this build release. + # + # Defaults to { + # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": , + # } + soong_env: typing.Dict[str, str] = None + + # The sdk versions that need to be generated for this build release. + sdk_versions: List[str] = \ + dataclasses.field(default_factory=lambda: SDK_VERSIONS) + + # The position of this instance within the BUILD_RELEASES list. + ordinal: int = dataclasses.field(default=-1, init=False) + + def __post_init__(self): + # The following use object.__setattr__ as this object is frozen and + # attempting to set the fields directly would cause an exception to be + # thrown. + object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) + # Add this to the end of the list of all build releases. + ALL_BUILD_RELEASES.append(self) + # If no sub_dir was specified then set the default. + if self.sub_dir is None: + object.__setattr__(self, "sub_dir", f"for-{self.name}-build") + # If no soong_env was specified then set the default. + if self.soong_env is None: + object.__setattr__( + self, + "soong_env", + { + # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a + # snapshot suitable for a specific target build release. + "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, + }) + + def __le__(self, other): + return self.ordinal <= other.ordinal + + +def create_no_dist_snapshot(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """A place holder dist snapshot creation function that does nothing.""" + print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") + return + + +def create_sdk_snapshots_in_Soong(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """Builds sdks and populates the dist.""" + producer.produce_dist_for_build_release(build_release, modules) + return + + +def reuse_latest_sdk_snapshots(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """Copies the snapshots from the latest build.""" + producer.populate_dist(build_release, build_release.sdk_versions, modules) + return + + +Q = BuildRelease( + name="Q", + # At the moment we do not generate a snapshot for Q. + creator=create_no_dist_snapshot, +) +R = BuildRelease( + name="R", + # At the moment we do not generate a snapshot for R. + creator=create_no_dist_snapshot, +) +S = BuildRelease( + name="S", + # Generate a snapshot for S using Soong. + creator=create_sdk_snapshots_in_Soong, +) + +# Insert additional BuildRelease definitions for following releases here, +# before LATEST. + +# The build release for the latest build supported by this build, i.e. the +# current build. This must be the last BuildRelease defined in this script, +# before LEGACY_BUILD_RELEASE. +LATEST = BuildRelease( + name="latest", + creator=create_sdk_snapshots_in_Soong, + # There are no build release specific environment variables to pass to + # Soong. + soong_env={}, +) + +# The build release to populate the legacy dist structure that does not specify +# a particular build release. This MUST come after LATEST so that it includes +# all the modules for which sdk snapshot source is available. +LEGACY_BUILD_RELEASE = BuildRelease( + name="legacy", + # There is no build release specific sub directory. + sub_dir="", + # There are no build release specific environment variables to pass to + # Soong. + soong_env={}, + # Do not create new snapshots, simply use the snapshots generated for + # latest. + creator=reuse_latest_sdk_snapshots, +) + + @dataclasses.dataclass(frozen=True) class MainlineModule: """Represents a mainline module""" @@ -265,6 +414,19 @@ class MainlineModule: # The names of the sdk and module_exports. sdks: list[str] + # The first build release in which the SDK snapshot for this module is + # needed. + # + # Note: This is not necessarily the same build release in which the SDK + # source was first included. So, a module that was added in build T + # could potentially be used in an S release and so its SDK will need + # to be made available for S builds. + # + # Defaults to the latest build, i.e. the build on which this script is run + # as the snapshot is assumed to be needed in the build containing the sdk + # source. + first_release: BuildRelease = LATEST + # The configuration variable, defaults to ANDROID:module_build_from_source configVar: ConfigVar = ConfigVar( namespace="ANDROID", @@ -288,6 +450,10 @@ class MainlineModule: configBpDefFile=self.configBpDefFile), ] + def is_required_for(self, target_build_release): + """True if this module is required for the target build release.""" + return self.first_release <= target_build_release + # List of mainline modules. MAINLINE_MODULES = [ @@ -298,6 +464,7 @@ MAINLINE_MODULES = [ "art-module-test-exports", "art-module-host-exports", ], + first_release=S, # Override the config... fields. configVar=ConfigVar( namespace="art_module", @@ -313,50 +480,50 @@ MAINLINE_MODULES = [ "conscrypt-module-test-exports", "conscrypt-module-host-exports", ], + first_release=Q, ), MainlineModule( apex="com.android.ipsec", sdks=["ipsec-module-sdk"], + first_release=S, ), MainlineModule( apex="com.android.media", sdks=["media-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.mediaprovider", sdks=["mediaprovider-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.permission", sdks=["permission-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.sdkext", sdks=["sdkextensions-sdk"], + first_release=R, ), MainlineModule( apex="com.android.os.statsd", sdks=["statsd-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.tethering", sdks=["tethering-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.wifi", sdks=["wifi-module-sdk"], + first_release=R, ), ] -# A list of the sdk versions to build. Usually just current but can include a -# numeric version too. -SDK_VERSIONS = [ - # Suitable for overriding the source modules with prefer:true. - # Unlike "unversioned" this mode also adds "@current" suffixed modules - # with the same prebuilts (which are never preferred). - "current", -] - @dataclasses.dataclass class SdkDistProducer: @@ -381,14 +548,39 @@ class SdkDistProducer: # transformed to document where the changes came from. script: str = sys.argv[0] - def produce_dist(self, modules): - sdk_versions = SDK_VERSIONS - self.build_sdks(sdk_versions, modules) - self.populate_dist(sdk_versions, modules) + # The path to the mainline-sdks dist directory. + # + # Initialized in __post_init__(). + mainline_sdks_dir: str = dataclasses.field(init=False) + + def __post_init__(self): + self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") + + def prepare(self): + # Clear the mainline-sdks dist directory. + shutil.rmtree(self.mainline_sdks_dir, ignore_errors=True) + + def produce_dist(self, modules, build_releases): + # Prepare the dist directory for the sdks. + self.prepare() + + for build_release in build_releases: + # Only build modules that are required for this build release. + filtered_modules = [ + m for m in modules if m.is_required_for(build_release) + ] + if filtered_modules: + print(f"Building SDK snapshots for {build_release.name}" + f" build release") + build_release.creator(build_release, self, filtered_modules) + self.populate_stubs(modules) - def build_sdks(self, sdk_versions, modules): - self.snapshot_builder.build_snapshots(sdk_versions, modules) + def produce_dist_for_build_release(self, build_release, modules): + sdk_versions = build_release.sdk_versions + self.snapshot_builder.build_snapshots(build_release, sdk_versions, + modules) + self.populate_dist(build_release, sdk_versions, modules) def unzip_current_stubs(self, sdk_name, apex_name): """Unzips stubs for "current" into {producer.dist_dir}/stubs/{apex}.""" @@ -414,10 +606,9 @@ class SdkDistProducer: if sdk.endswith("-sdk"): self.unzip_current_stubs(sdk, apex) - def populate_dist(self, sdk_versions, modules): - # Clear and populate the mainline-sdks dist directory. - sdks_dist_dir = os.path.join(self.dist_dir, "mainline-sdks") - shutil.rmtree(sdks_dist_dir, ignore_errors=True) + def populate_dist(self, build_release, sdk_versions, modules): + build_release_dist_dir = os.path.join(self.mainline_sdks_dir, + build_release.sub_dir) for module in modules: apex = module.apex @@ -431,8 +622,8 @@ class SdkDistProducer: f" ^[^-]+-(module-)?(sdk|host-exports|test-exports)" ) - sdk_dist_dir = os.path.join(sdks_dist_dir, sdk_version, - apex, subdir) + sdk_dist_dir = os.path.join(build_release_dist_dir, + sdk_version, apex, subdir) sdk_path = self.snapshot_builder.get_sdk_path( sdk, sdk_version) self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_dir, @@ -559,7 +750,8 @@ def main(): producer = create_producer() modules = filter_modules(MAINLINE_MODULES) - producer.produce_dist(modules) + + producer.produce_dist(modules, ALL_BUILD_RELEASES) if __name__ == "__main__": diff --git a/build/mainline_modules_sdks_test.py b/build/mainline_modules_sdks_test.py index cf6b28c..d4d6faa 100644 --- a/build/mainline_modules_sdks_test.py +++ b/build/mainline_modules_sdks_test.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for mainline_modules_sdks.py.""" +import dataclasses from pathlib import Path import os import tempfile @@ -42,7 +43,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder): z.writestr("sdk_library/public/lib.jar", "") z.writestr("sdk_library/public/api.txt", "") - def build_snapshots(self, sdk_versions, modules): + def build_snapshots(self, build_release, sdk_versions, modules): # Create input file structure. sdks_out_dir = Path(self.get_mainline_sdks_path()) sdks_out_dir.mkdir(parents=True, exist_ok=True) @@ -75,13 +76,21 @@ class TestProduceDist(unittest.TestCase): out_dir=tmp_out_dir, ) + build_releases = [ + mm.Q, + mm.R, + mm.S, + mm.LATEST, + mm.LEGACY_BUILD_RELEASE, + ] + producer = mm.SdkDistProducer( subprocess_runner=subprocess_runner, snapshot_builder=snapshot_builder, dist_dir=tmp_dist_dir, ) - producer.produce_dist(modules) + producer.produce_dist(modules, build_releases) files = [] for abs_dir, _, filenames in os.walk(tmp_dist_dir): @@ -89,20 +98,33 @@ class TestProduceDist(unittest.TestCase): for f in filenames: files.append(os.path.join(rel_dir, f)) # pylint: disable=line-too-long - self.assertEqual([ - "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip", - "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip", - "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip", - "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", - "stubs/com.android.art/sdk_library/public/api.txt", - "stubs/com.android.art/sdk_library/public/lib.jar", - "stubs/com.android.art/sdk_library/public/removed.txt", - "stubs/com.android.art/sdk_library/public/source.srcjar", - "stubs/com.android.ipsec/sdk_library/public/api.txt", - "stubs/com.android.ipsec/sdk_library/public/lib.jar", - "stubs/com.android.ipsec/sdk_library/public/removed.txt", - "stubs/com.android.ipsec/sdk_library/public/source.srcjar", - ], sorted(files)) + self.assertEqual( + [ + # Legacy copy of the snapshots, for use by tools that don't support build specific snapshots. + "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + # Build specific snapshots. + "mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + # Legacy stubs directory containing unpacked java_sdk_library artifacts. + "stubs/com.android.art/sdk_library/public/api.txt", + "stubs/com.android.art/sdk_library/public/lib.jar", + "stubs/com.android.art/sdk_library/public/removed.txt", + "stubs/com.android.art/sdk_library/public/source.srcjar", + "stubs/com.android.ipsec/sdk_library/public/api.txt", + "stubs/com.android.ipsec/sdk_library/public/lib.jar", + "stubs/com.android.ipsec/sdk_library/public/removed.txt", + "stubs/com.android.ipsec/sdk_library/public/source.srcjar", + ], + sorted(files)) def pathToTestData(relative_path):