Files
android_packages_modules_co…/build/mainline_modules_sdks_test.py
Paul Duffin f3c17ef416 Retry: Generate soong_config_module_type inline in the snapshot
Previously, this imported the soong_config_module_type definitions from
a manually curated file. That had a couple of problems:
1. It required that the file be manually updated everytime that a new
   module_type was added to an sdk snapshot controlled by a specific
   config variable.
2. Adding a new config variable required creating another file to be
   manually curated.
3. It made a snapshot that was generated for an earlier release
   dependent on an additional file from that release which was in a
   separate repository and would require a lot of extra work to update
   if necessary.

This change generates the soong_config_module_type inline in the
snapshot Android.bp file and while it makes it slightly bigger it makes
each snapshot much more self contained and makes it easier to add
module specific config variables.

The latter is the main driver for this change as follow up changes will
add module specific variables for the optional modules.

Due to bug 235475711 each soong_config_module_type has to be unique
across all mainline modules so this uses both the module short name
and a prefix associated with the SdkType to generate unique module
names. However, that only affects the S snapshots as from T onwards is
is not necessary to use soong_config_module_type modules at all.

Bug: 233965247
Test: atest --host mainline_modules_sdks_test
      packages/modules/common/build/mainline_modules_sdks.sh
      # Extract art sdk and module_exports snapshots into
      # prebuilts/module_sdk/art and then run "m nothing"
Change-Id: Ib0e2ece1779451dffff6dfaca7ca39b264004b5b
2022-07-26 16:40:25 +00:00

558 lines
23 KiB
Python

#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for mainline_modules_sdks.py."""
import dataclasses
import re
import typing
from pathlib import Path
import os
import shutil
import tempfile
import unittest
import zipfile
from unittest import mock
import mainline_modules_sdks as mm
MAINLINE_MODULES_BY_APEX = dict(
(m.apex, m) for m in (mm.MAINLINE_MODULES + mm.BUNDLED_MAINLINE_MODULES +
mm.PLATFORM_SDKS_FOR_MAINLINE))
@dataclasses.dataclass()
class FakeSnapshotBuilder(mm.SnapshotBuilder):
"""A fake snapshot builder that does not run the build.
This skips the whole build process and just creates some fake sdk
modules.
"""
snapshots: typing.List[typing.Any] = dataclasses.field(default_factory=list)
@staticmethod
def create_sdk_library_files(z, name):
z.writestr(f"sdk_library/public/{name}-removed.txt", "")
z.writestr(f"sdk_library/public/{name}.srcjar", "")
z.writestr(f"sdk_library/public/{name}-stubs.jar", "")
z.writestr(f"sdk_library/public/{name}.txt",
"method public int testMethod(int);")
def create_snapshot_file(self, out_dir, name, version, for_r_build):
zip_file = Path(mm.sdk_snapshot_zip_file(out_dir, name, version))
with zipfile.ZipFile(zip_file, "w") as z:
z.writestr("Android.bp", "")
if name.endswith("-sdk"):
if for_r_build:
for library in for_r_build.sdk_libraries:
self.create_sdk_library_files(z, library.name)
else:
self.create_sdk_library_files(z, re.sub(r"-.*$", "", name))
def build_snapshots(self, build_release, sdk_versions, modules):
self.snapshots.append((build_release.name, build_release.soong_env,
sdk_versions, [m.apex for m in modules]))
# Create input file structure.
sdks_out_dir = Path(self.mainline_sdks_dir).joinpath("test")
sdks_out_dir.mkdir(parents=True, exist_ok=True)
# Create a fake sdk zip file for each module.
for module in modules:
for sdk in module.sdks:
for sdk_version in sdk_versions:
self.create_snapshot_file(sdks_out_dir, sdk, sdk_version,
module.for_r_build)
return sdks_out_dir
def get_art_module_info_file_data(self, sdk):
info_file_data = f"""[
{{
"@type": "java_sdk_library",
"@name": "art.module.public.api",
"@deps": [
"libcore_license"
],
"dist_stem": "art",
"scopes": {{
"public": {{
"current_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}.txt",
"latest_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art.api.public.latest/gen/art.api.public.latest",
"latest_removed_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art-removed.api.public.latest/gen/art-removed.api.public.latest",
"removed_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}-removed.txt"
}}
}}
}}
]
"""
return info_file_data
@staticmethod
def write_data_to_file(file, data):
with open(file, "w", encoding="utf8") as fd:
fd.write(data)
def create_snapshot_info_file(self, module, sdk_info_file, sdk):
if module == MAINLINE_MODULES_BY_APEX["com.android.art"]:
self.write_data_to_file(sdk_info_file,
self.get_art_module_info_file_data(sdk))
else:
# For rest of the modules, generate an empty .info file.
self.write_data_to_file(sdk_info_file, "[]")
def build_sdk_scope_targets(self, build_release, sdk_version, modules):
target_paths = []
target_dict = {}
for module in modules:
for sdk in module.sdks:
if "host-exports" in sdk or "test-exports" in sdk:
continue
sdk_info_file = mm.sdk_snapshot_info_file(
Path(self.mainline_sdks_dir).joinpath("test"), sdk,
sdk_version)
self.create_snapshot_info_file(module, sdk_info_file, sdk)
paths, dict_item = self.latest_api_file_targets(sdk_info_file)
target_paths.extend(paths)
target_dict[sdk_info_file] = dict_item
for target_path in target_paths:
os.makedirs(os.path.split(target_path)[0])
self.write_data_to_file(target_path, "")
return target_dict
class TestProduceDist(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.tmp_out_dir = os.path.join(self.tmp_dir, "out")
os.mkdir(self.tmp_out_dir)
self.tmp_dist_dir = os.path.join(self.tmp_dir, "dist")
os.mkdir(self.tmp_dist_dir)
def tearDown(self):
shutil.rmtree(self.tmp_dir, ignore_errors=True)
def produce_dist(self, modules, build_releases):
subprocess_runner = mm.SubprocessRunner()
snapshot_builder = FakeSnapshotBuilder(
tool_path="path/to/mainline_modules_sdks.sh",
subprocess_runner=subprocess_runner,
out_dir=self.tmp_out_dir,
)
producer = mm.SdkDistProducer(
subprocess_runner=subprocess_runner,
snapshot_builder=snapshot_builder,
dist_dir=self.tmp_dist_dir,
)
producer.produce_dist(modules, build_releases)
def list_files_in_dir(self, tmp_dist_dir):
files = []
for abs_dir, _, filenames in os.walk(tmp_dist_dir):
rel_dir = os.path.relpath(abs_dir, tmp_dist_dir)
if rel_dir == ".":
rel_dir = ""
for f in filenames:
files.append(os.path.join(rel_dir, f))
return files
def test_unbundled_modules(self):
# Create the out/soong/build_number.txt file that is copied into the
# snapshots.
self.create_build_number_file()
modules = [
MAINLINE_MODULES_BY_APEX["com.android.art"],
MAINLINE_MODULES_BY_APEX["com.android.ipsec"],
# Create a google specific module.
mm.aosp_to_google(MAINLINE_MODULES_BY_APEX["com.android.wifi"]),
]
build_releases = [
mm.Q,
mm.R,
mm.S,
mm.LATEST,
]
self.produce_dist(modules, build_releases)
# pylint: disable=line-too-long
self.assertEqual(
[
# Build specific snapshots.
"mainline-sdks/for-R-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-R-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
],
sorted(self.list_files_in_dir(self.tmp_dist_dir)))
r_snaphot_dir = os.path.join(self.tmp_out_dir,
"soong/mainline-sdks/test/for-R-build")
aosp_ipsec_r_bp_file = "com.android.ipsec/sdk_library/Android.bp"
google_wifi_android_bp = "com.google.android.wifi/sdk_library/Android.bp"
self.assertEqual([
aosp_ipsec_r_bp_file,
"com.android.ipsec/sdk_library/public/android.net.ipsec.ike-removed.txt",
"com.android.ipsec/sdk_library/public/android.net.ipsec.ike-stubs.jar",
"com.android.ipsec/sdk_library/public/android.net.ipsec.ike.srcjar",
"com.android.ipsec/sdk_library/public/android.net.ipsec.ike.txt",
"com.android.ipsec/snapshot-creation-build-number.txt",
google_wifi_android_bp,
"com.google.android.wifi/sdk_library/public/framework-wifi-removed.txt",
"com.google.android.wifi/sdk_library/public/framework-wifi-stubs.jar",
"com.google.android.wifi/sdk_library/public/framework-wifi.srcjar",
"com.google.android.wifi/sdk_library/public/framework-wifi.txt",
"com.google.android.wifi/snapshot-creation-build-number.txt",
"ipsec-module-sdk-current.zip",
"wifi-module-sdk-current.zip",
], sorted(self.list_files_in_dir(r_snaphot_dir)))
def read_r_snapshot_contents(path):
abs_path = os.path.join(r_snaphot_dir, path)
with open(abs_path, "r", encoding="utf8") as file:
return file.read()
# Check the contents of the AOSP ipsec module
ipsec_contents = read_r_snapshot_contents(aosp_ipsec_r_bp_file)
expected = read_test_data("ipsec_for_r_Android.bp")
self.assertEqual(expected, ipsec_contents)
# Check the contents of the Google ipsec module
wifi_contents = read_r_snapshot_contents(google_wifi_android_bp)
expected = read_test_data("google_wifi_for_r_Android.bp")
self.assertEqual(expected, wifi_contents)
def test_old_release(self):
modules = [
MAINLINE_MODULES_BY_APEX["com.android.art"], # An unnbundled module
MAINLINE_MODULES_BY_APEX["com.android.runtime"], # A bundled module
MAINLINE_MODULES_BY_APEX["platform-mainline"], # Platform SDK
]
build_releases = [mm.S]
self.produce_dist(modules, build_releases)
# pylint: disable=line-too-long
self.assertEqual([
"mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
], sorted(self.list_files_in_dir(self.tmp_dist_dir)))
def test_latest_release(self):
modules = [
MAINLINE_MODULES_BY_APEX["com.android.art"], # An unnbundled module
MAINLINE_MODULES_BY_APEX["com.android.runtime"], # A bundled module
MAINLINE_MODULES_BY_APEX["platform-mainline"], # Platform SDK
]
build_releases = [mm.LATEST]
self.produce_dist(modules, build_releases)
# pylint: disable=line-too-long
self.assertEqual(
[
# Bundled modules and platform SDKs.
"bundled-mainline-sdks/com.android.runtime/host-exports/runtime-module-host-exports-current.zip",
"bundled-mainline-sdks/com.android.runtime/sdk/runtime-module-sdk-current.zip",
"bundled-mainline-sdks/platform-mainline/sdk/platform-mainline-sdk-current.zip",
"bundled-mainline-sdks/platform-mainline/test-exports/platform-mainline-test-exports-current.zip",
# Unbundled (normal) modules.
"mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
],
sorted(self.list_files_in_dir(self.tmp_dist_dir)))
art_api_diff_file = os.path.join(
self.tmp_dist_dir,
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt"
)
self.assertNotEqual(
os.path.getsize(art_api_diff_file),
0,
msg="Api diff file should not be empty for the art module")
def create_build_number_file(self):
soong_dir = os.path.join(self.tmp_out_dir, "soong")
os.makedirs(soong_dir, exist_ok=True)
build_number_file = os.path.join(soong_dir, "build_number.txt")
with open(build_number_file, "w", encoding="utf8") as f:
f.write("build-number")
def test_snapshot_build_order(self):
# Create the out/soong/build_number.txt file that is copied into the
# snapshots.
self.create_build_number_file()
subprocess_runner = unittest.mock.Mock(mm.SubprocessRunner)
snapshot_builder = FakeSnapshotBuilder(
tool_path="path/to/mainline_modules_sdks.sh",
subprocess_runner=subprocess_runner,
out_dir=self.tmp_out_dir,
)
producer = mm.SdkDistProducer(
subprocess_runner=subprocess_runner,
snapshot_builder=snapshot_builder,
dist_dir=self.tmp_dist_dir,
)
modules = [
MAINLINE_MODULES_BY_APEX["com.android.art"],
MAINLINE_MODULES_BY_APEX["com.android.ipsec"],
# Create a google specific module.
mm.aosp_to_google(MAINLINE_MODULES_BY_APEX["com.android.wifi"]),
]
build_releases = [
mm.Q,
mm.R,
mm.S,
mm.LATEST,
]
producer.produce_dist(modules, build_releases)
# Check the order in which the snapshots are built.
self.assertEqual([
(
"R",
{},
["current"],
["com.android.ipsec", "com.google.android.wifi"],
),
(
"latest",
{},
["current"],
[
"com.android.art", "com.android.ipsec",
"com.google.android.wifi"
],
),
(
"S",
{
"SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": "S"
},
["current"],
[
"com.android.art", "com.android.ipsec",
"com.google.android.wifi"
],
),
], snapshot_builder.snapshots)
def path_to_test_data(relative_path):
"""Construct a path to a test data file.
The relative_path is relative to the location of this file.
"""
this_file = __file__
# When running as a python_test_host (name=<x>) with an embedded launcher
# the __file__ points to .../<x>/<x>.py but the .../<x> is not a directory
# it is a binary with the launcher and the python file embedded inside. In
# that case a test data file <rel> is at .../<x>_data/<rel>, not
# .../<x>/<x>_data/<rel> so it is necessary to trim the base name (<x>.py)
# from the file.
if not os.path.isfile(this_file):
this_file = os.path.dirname(this_file)
# When the python file is at .../<x>.py (or in the case of an embedded
# launcher at .../<x>/<x>.py) then the test data is at .../<x>_data/<rel>.
this_file_without_ext, _ = os.path.splitext(this_file)
return os.path.join(this_file_without_ext + "_data", relative_path)
def read_test_data(relative_path):
with open(path_to_test_data(relative_path), "r", encoding="utf8") as f:
return f.read()
class TestAndroidBpTransformations(unittest.TestCase):
def apply_transformations(self, src, transformations, expected):
producer = mm.SdkDistProducer(
subprocess_runner=mock.Mock(mm.SubprocessRunner),
snapshot_builder=mock.Mock(mm.SnapshotBuilder),
script=self._testMethodName,
)
with tempfile.TemporaryDirectory() as tmp_dir:
path = os.path.join(tmp_dir, "Android.bp")
with open(path, "w", encoding="utf8") as f:
f.write(src)
mm.apply_transformations(producer, tmp_dir, transformations)
with open(path, "r", encoding="utf8") as f:
result = f.read()
self.maxDiff = None
self.assertEqual(expected, result)
def test_common_mainline_module(self):
"""Tests the transformations applied to a common mainline sdk on S.
This uses ipsec as an example of a common mainline sdk. This checks
that the general Soong config module types and variables are used.
"""
src = read_test_data("ipsec_Android.bp.input")
expected = read_test_data("ipsec_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.ipsec"]
transformations = module.transformations(mm.S, mm.Sdk)
self.apply_transformations(src, transformations, expected)
def test_common_mainline_module_tiramisu(self):
"""Tests the transformations applied to a common mainline sdk on T.
This uses ipsec as an example of a common mainline sdk. This checks
that the use_source_config_var property is inserted.
"""
src = read_test_data("ipsec_Android.bp.input")
expected = read_test_data("ipsec_tiramisu_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.ipsec"]
transformations = module.transformations(mm.Tiramisu, mm.Sdk)
self.apply_transformations(src, transformations, expected)
def test_optional_mainline_module(self):
"""Tests the transformations applied to an optional mainline sdk on S.
This uses wifi as an example of a optional mainline sdk. This checks
that the module specific Soong config module types and variables are
used.
"""
src = read_test_data("wifi_Android.bp.input")
expected = read_test_data("wifi_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.wifi"]
transformations = module.transformations(mm.S, mm.Sdk)
self.apply_transformations(src, transformations, expected)
def test_optional_mainline_module_tiramisu(self):
"""Tests the transformations applied to an optional mainline sdk on T.
This uses wifi as an example of a optional mainline sdk. This checks
that the use_source_config_var property is inserted.
"""
src = read_test_data("wifi_Android.bp.input")
expected = read_test_data("wifi_tiramisu_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.wifi"]
transformations = module.transformations(mm.Tiramisu, mm.Sdk)
self.apply_transformations(src, transformations, expected)
def test_art(self):
"""Tests the transformations applied to a the ART mainline module.
The ART mainline module uses a different Soong config setup to the
common mainline modules. This checks that the ART specific Soong config
module types, and variables are used.
"""
src = read_test_data("art_Android.bp.input")
expected = read_test_data("art_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.art"]
transformations = module.transformations(mm.S, mm.Sdk)
self.apply_transformations(src, transformations, expected)
def test_art_module_exports(self):
"""Tests the transformations applied to a the ART mainline module.
The ART mainline module uses a different Soong config setup to the
common mainline modules. This checks that the ART specific Soong config
module types, and variables are used.
"""
src = read_test_data("art_Android.bp.input")
expected = read_test_data("art_host_exports_Android.bp.expected")
module = MAINLINE_MODULES_BY_APEX["com.android.art"]
transformations = module.transformations(mm.S, mm.HostExports)
self.apply_transformations(src, transformations, expected)
def test_r_build(self):
"""Tests the transformations that are applied for the R build.
This uses ipsec as an example of a common mainline module. That would
usually apply the mm.SoongConfigBoilerplateInserter transformation but
because this is being run for build R that transformation should not be
applied.
"""
src = read_test_data("ipsec_for_r_Android.bp")
# There should be no changes made.
expected = src
module = MAINLINE_MODULES_BY_APEX["com.android.ipsec"]
transformations = module.transformations(mm.R, mm.Sdk)
self.apply_transformations(src, transformations, expected)
class TestFilterModules(unittest.TestCase):
def test_no_filter(self):
all_modules = mm.MAINLINE_MODULES + mm.BUNDLED_MAINLINE_MODULES
modules = mm.filter_modules(all_modules, None)
self.assertEqual(modules, all_modules)
def test_with_filter(self):
modules = mm.filter_modules(mm.MAINLINE_MODULES, "com.android.art")
expected = MAINLINE_MODULES_BY_APEX["com.android.art"]
self.assertEqual(modules, [expected])
class TestModuleProperties(unittest.TestCase):
def test_unbundled(self):
for module in mm.MAINLINE_MODULES:
with self.subTest(module=module):
self.assertFalse(module.is_bundled())
def test_bundled(self):
for module in (mm.BUNDLED_MAINLINE_MODULES +
mm.PLATFORM_SDKS_FOR_MAINLINE):
with self.subTest(module=module):
self.assertTrue(module.is_bundled())
self.assertEqual(module.first_release, mm.LATEST)
if __name__ == "__main__":
unittest.main(verbosity=2)