LineageOS/android_packages_modules_common
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support building build target specific snapshots

Browse Source 
This changes adds support for building target build release specific
snapshots.

It adds a BuildRelease class that defines the characteristics of the
build release, e.g. name, how to create it, etc. It also adds a
first_release field to MainlineModule which specifies the first
BuildRelease for which a snapshot of the module is required and
initializes that field for each module.

After these changes this script will generate:
1. A legacy set of snapshots that match the file structure that was
   generated without this change. This is intended to allow existing
   consumers of the generated artifacts to continue to work while they
   are modified to make use of target build release specific snapshots.

2. The set of snapshots for the latest build release, i.e. the build
   release containing the source from which the snapshots are produced.
   This includes snapshots for all the modules.

3. For each build release from S onwards a set of snapshots for the
   modules it supports.

It does not currently generate snapshots for Q and R releases as Soong
cannot generate a compatible snapshot for them. If it is necessary to
generate snapshots for those target build releases (similar to what the
packages/modules/common/generate_ml_bundle.sh would produce) then a
follow up change will add that capability to this script.

Bug: 204763318
Test: atest mainline_modules_sdks_test
      packages/modules/common/build/mainline_modules_sdks.sh
      tree out/dist/mainline-sdks out/dist/stubs
      - check that it adds the for-latest-build directory but is
        otherwise identical to what was produced before this change.
Change-Id: I48eb0b69cbe8664106b826ee0898557c98e039c2
This commit is contained in:
Paul Duffin
2022-01-17 16:39:00 +00:00
parent 6ea0eb45c5
commit 01ea57c593
2 changed files with 258 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

248
build/mainline_modules_sdks.py
View File

@@ -27,6 +27,8 @@ import shutil
import subprocess
import sys
import tempfile
import typing
from typing import Callable, List
import zipfile
@@ -214,7 +216,7 @@ class SnapshotBuilder:
return os.path.join(self.get_mainline_sdks_path(),
f"{sdk_name}-{sdk_version}.zip")
def build_snapshots(self, sdk_versions, modules):
def build_snapshots(self, build_release, sdk_versions, modules):
# Build the SDKs once for each version.
for sdk_version in sdk_versions:
# Compute the paths to all the Soong generated sdk snapshot files
@@ -225,16 +227,20 @@ class SnapshotBuilder:
for sdk in module.sdks
]
# TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but we
# currently break without it.
#
# Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside sdk
# zip files as expected by prebuilt drop.
# Extra environment variables to pass to the build process.
extraEnv = {
# TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but
# we currently break without it.
"SOONG_ALLOW_MISSING_DEPENDENCIES": "true",
# Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside
# sdk zip files as expected by prebuilt drop.
"SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true",
# Set SOONG_SDK_SNAPSHOT_VERSION to generate the appropriately
# tagged version of the sdk.
"SOONG_SDK_SNAPSHOT_VERSION": sdk_version,
}
extraEnv.update(build_release.soong_env)
# Unless explicitly specified in the calling environment set
# TARGET_BUILD_VARIANT=user.
# This MUST be identical to the TARGET_BUILD_VARIANT used to build
@@ -256,6 +262,149 @@ class SnapshotBuilder:
self.subprocess_runner.run(cmd, env=env)
# A list of the sdk versions to build. Usually just current but can include a
# numeric version too.
SDK_VERSIONS = [
# Suitable for overriding the source modules with prefer:true.
# Unlike "unversioned" this mode also adds "@current" suffixed modules
# with the same prebuilts (which are never preferred).
"current",
# Insert additional sdk versions needed for the latest build release.
]
# The initially empty list of build releases. Every BuildRelease that is created
# automatically appends itself to this list.
ALL_BUILD_RELEASES = []
@dataclasses.dataclass(frozen=True)
class BuildRelease:
"""Represents a build release"""
# The name of the build release, e.g. Q, R, S, T, etc.
name: str
# The function to call to create the snapshot in the dist, that covers
# building and copying the snapshot into the dist.
creator: Callable[
["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None]
# The sub-directory of dist/mainline-sdks into which the build release
# specific snapshots will be copied.
#
# Defaults to for-<name>-build.
sub_dir: str = None
# Additional environment variables to pass to Soong when building the
# snapshots for this build release.
#
# Defaults to {
# "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>,
# }
soong_env: typing.Dict[str, str] = None
# The sdk versions that need to be generated for this build release.
sdk_versions: List[str] = \
dataclasses.field(default_factory=lambda: SDK_VERSIONS)
# The position of this instance within the BUILD_RELEASES list.
ordinal: int = dataclasses.field(default=-1, init=False)
def __post_init__(self):
# The following use object.__setattr__ as this object is frozen and
# attempting to set the fields directly would cause an exception to be
# thrown.
object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES))
# Add this to the end of the list of all build releases.
ALL_BUILD_RELEASES.append(self)
# If no sub_dir was specified then set the default.
if self.sub_dir is None:
object.__setattr__(self, "sub_dir", f"for-{self.name}-build")
# If no soong_env was specified then set the default.
if self.soong_env is None:
object.__setattr__(
self,
"soong_env",
{
# Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a
# snapshot suitable for a specific target build release.
"SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name,
})
def __le__(self, other):
return self.ordinal <= other.ordinal
def create_no_dist_snapshot(build_release: BuildRelease,
producer: "SdkDistProducer",
modules: List["MainlineModule"]):
"""A place holder dist snapshot creation function that does nothing."""
print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}")
return
def create_sdk_snapshots_in_Soong(build_release: BuildRelease,
producer: "SdkDistProducer",
modules: List["MainlineModule"]):
"""Builds sdks and populates the dist."""
producer.produce_dist_for_build_release(build_release, modules)
return
def reuse_latest_sdk_snapshots(build_release: BuildRelease,
producer: "SdkDistProducer",
modules: List["MainlineModule"]):
"""Copies the snapshots from the latest build."""
producer.populate_dist(build_release, build_release.sdk_versions, modules)
return
Q = BuildRelease(
name="Q",
# At the moment we do not generate a snapshot for Q.
creator=create_no_dist_snapshot,
)
R = BuildRelease(
name="R",
# At the moment we do not generate a snapshot for R.
creator=create_no_dist_snapshot,
)
S = BuildRelease(
name="S",
# Generate a snapshot for S using Soong.
creator=create_sdk_snapshots_in_Soong,
)
# Insert additional BuildRelease definitions for following releases here,
# before LATEST.
# The build release for the latest build supported by this build, i.e. the
# current build. This must be the last BuildRelease defined in this script,
# before LEGACY_BUILD_RELEASE.
LATEST = BuildRelease(
name="latest",
creator=create_sdk_snapshots_in_Soong,
# There are no build release specific environment variables to pass to
# Soong.
soong_env={},
)
# The build release to populate the legacy dist structure that does not specify
# a particular build release. This MUST come after LATEST so that it includes
# all the modules for which sdk snapshot source is available.
LEGACY_BUILD_RELEASE = BuildRelease(
name="legacy",
# There is no build release specific sub directory.
sub_dir="",
# There are no build release specific environment variables to pass to
# Soong.
soong_env={},
# Do not create new snapshots, simply use the snapshots generated for
# latest.
creator=reuse_latest_sdk_snapshots,
)
@dataclasses.dataclass(frozen=True)
class MainlineModule:
"""Represents a mainline module"""
@@ -265,6 +414,19 @@ class MainlineModule:
# The names of the sdk and module_exports.
sdks: list[str]
# The first build release in which the SDK snapshot for this module is
# needed.
#
# Note: This is not necessarily the same build release in which the SDK
# source was first included. So, a module that was added in build T
# could potentially be used in an S release and so its SDK will need
# to be made available for S builds.
#
# Defaults to the latest build, i.e. the build on which this script is run
# as the snapshot is assumed to be needed in the build containing the sdk
# source.
first_release: BuildRelease = LATEST
# The configuration variable, defaults to ANDROID:module_build_from_source
configVar: ConfigVar = ConfigVar(
namespace="ANDROID",
@@ -288,6 +450,10 @@ class MainlineModule:
configBpDefFile=self.configBpDefFile),
]
def is_required_for(self, target_build_release):
"""True if this module is required for the target build release."""
return self.first_release <= target_build_release
# List of mainline modules.
MAINLINE_MODULES = [
@@ -298,6 +464,7 @@ MAINLINE_MODULES = [
"art-module-test-exports",
"art-module-host-exports",
],
first_release=S,
# Override the config... fields.
configVar=ConfigVar(
namespace="art_module",
@@ -313,50 +480,50 @@ MAINLINE_MODULES = [
"conscrypt-module-test-exports",
"conscrypt-module-host-exports",
],
first_release=Q,
),
MainlineModule(
apex="com.android.ipsec",
sdks=["ipsec-module-sdk"],
first_release=S,
),
MainlineModule(
apex="com.android.media",
sdks=["media-module-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.mediaprovider",
sdks=["mediaprovider-module-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.permission",
sdks=["permission-module-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.sdkext",
sdks=["sdkextensions-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.os.statsd",
sdks=["statsd-module-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.tethering",
sdks=["tethering-module-sdk"],
first_release=R,
),
MainlineModule(
apex="com.android.wifi",
sdks=["wifi-module-sdk"],
first_release=R,
),
]
# A list of the sdk versions to build. Usually just current but can include a
# numeric version too.
SDK_VERSIONS = [
# Suitable for overriding the source modules with prefer:true.
# Unlike "unversioned" this mode also adds "@current" suffixed modules
# with the same prebuilts (which are never preferred).
"current",
]
@dataclasses.dataclass
class SdkDistProducer:
@@ -381,14 +548,39 @@ class SdkDistProducer:
# transformed to document where the changes came from.
script: str = sys.argv[0]
def produce_dist(self, modules):
sdk_versions = SDK_VERSIONS
self.build_sdks(sdk_versions, modules)
self.populate_dist(sdk_versions, modules)
# The path to the mainline-sdks dist directory.
#
# Initialized in __post_init__().
mainline_sdks_dir: str = dataclasses.field(init=False)
def __post_init__(self):
self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks")
def prepare(self):
# Clear the mainline-sdks dist directory.
shutil.rmtree(self.mainline_sdks_dir, ignore_errors=True)
def produce_dist(self, modules, build_releases):
# Prepare the dist directory for the sdks.
self.prepare()
for build_release in build_releases:
# Only build modules that are required for this build release.
filtered_modules = [
m for m in modules if m.is_required_for(build_release)
]
if filtered_modules:
print(f"Building SDK snapshots for {build_release.name}"
f" build release")
build_release.creator(build_release, self, filtered_modules)
self.populate_stubs(modules)
def build_sdks(self, sdk_versions, modules):
self.snapshot_builder.build_snapshots(sdk_versions, modules)
def produce_dist_for_build_release(self, build_release, modules):
sdk_versions = build_release.sdk_versions
self.snapshot_builder.build_snapshots(build_release, sdk_versions,
modules)
self.populate_dist(build_release, sdk_versions, modules)
def unzip_current_stubs(self, sdk_name, apex_name):
"""Unzips stubs for "current" into {producer.dist_dir}/stubs/{apex}."""
@@ -414,10 +606,9 @@ class SdkDistProducer:
if sdk.endswith("-sdk"):
self.unzip_current_stubs(sdk, apex)
def populate_dist(self, sdk_versions, modules):
# Clear and populate the mainline-sdks dist directory.
sdks_dist_dir = os.path.join(self.dist_dir, "mainline-sdks")
shutil.rmtree(sdks_dist_dir, ignore_errors=True)
def populate_dist(self, build_release, sdk_versions, modules):
build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
build_release.sub_dir)
for module in modules:
apex = module.apex
@@ -431,8 +622,8 @@ class SdkDistProducer:
f" ^[^-]+-(module-)?(sdk|host-exports|test-exports)"
)
sdk_dist_dir = os.path.join(sdks_dist_dir, sdk_version,
apex, subdir)
sdk_dist_dir = os.path.join(build_release_dist_dir,
sdk_version, apex, subdir)
sdk_path = self.snapshot_builder.get_sdk_path(
sdk, sdk_version)
self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_dir,
@@ -559,7 +750,8 @@ def main():
producer = create_producer()
modules = filter_modules(MAINLINE_MODULES)
producer.produce_dist(modules)
producer.produce_dist(modules, ALL_BUILD_RELEASES)
if __name__ == "__main__":

30
build/mainline_modules_sdks_test.py
View File

@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for mainline_modules_sdks.py."""
import dataclasses
from pathlib import Path
import os
import tempfile
@@ -42,7 +43,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
z.writestr("sdk_library/public/lib.jar", "")
z.writestr("sdk_library/public/api.txt", "")
def build_snapshots(self, sdk_versions, modules):
def build_snapshots(self, build_release, sdk_versions, modules):
# Create input file structure.
sdks_out_dir = Path(self.get_mainline_sdks_path())
sdks_out_dir.mkdir(parents=True, exist_ok=True)
@@ -75,13 +76,21 @@ class TestProduceDist(unittest.TestCase):
out_dir=tmp_out_dir,
)
build_releases = [
mm.Q,
mm.R,
mm.S,
mm.LATEST,
mm.LEGACY_BUILD_RELEASE,
]
producer = mm.SdkDistProducer(
subprocess_runner=subprocess_runner,
snapshot_builder=snapshot_builder,
dist_dir=tmp_dist_dir,
)
producer.produce_dist(modules)
producer.produce_dist(modules, build_releases)
files = []
for abs_dir, _, filenames in os.walk(tmp_dist_dir):
@@ -89,11 +98,23 @@ class TestProduceDist(unittest.TestCase):
for f in filenames:
files.append(os.path.join(rel_dir, f))
# pylint: disable=line-too-long
self.assertEqual([
self.assertEqual(
[
# Legacy copy of the snapshots, for use by tools that don't support build specific snapshots.
"mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
# Build specific snapshots.
"mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
# Legacy stubs directory containing unpacked java_sdk_library artifacts.
"stubs/com.android.art/sdk_library/public/api.txt",
"stubs/com.android.art/sdk_library/public/lib.jar",
"stubs/com.android.art/sdk_library/public/removed.txt",
@@ -102,7 +123,8 @@ class TestProduceDist(unittest.TestCase):
"stubs/com.android.ipsec/sdk_library/public/lib.jar",
"stubs/com.android.ipsec/sdk_library/public/removed.txt",
"stubs/com.android.ipsec/sdk_library/public/source.srcjar",
], sorted(files))
],
sorted(files))
def pathToTestData(relative_path):