LineageOS/android_packages_modules_common
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generating api diff files for each module sdk.

Browse Source 
In order to detect the addition of any new APIs in any mainline module,
an API diff file (current module sdk vs the last finalized module sdk)
needs to be generated correpsonding to each module sdk.

This CL generates the API diff files corresponding to each module sdk
and copy that to the out/dist directory.

This CL also contains the tests to verify the generation of api diff
files.

Bug: 230609867
Test: builds successfully. Generates API diff files for each module sdk
"atest mainline_modules_sdks_test" passes.

Change-Id: I1d8276a4054b92490ccca3dae36c48c639925f46
This commit is contained in:
Gurpreet Singh
2022-06-08 19:43:44 +00:00
parent 2c7c723d38
commit a61770fed2
2 changed files with 123 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
build/mainline_modules_sdks.py
View File

@@ -227,6 +227,11 @@ def sdk_snapshot_info_file(snapshots_dir, sdk_name, sdk_version):
return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}.info")
def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name, sdk_version):
"""Get the path to the sdk snapshot api diff file."""
return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}-api-diff.txt")
@dataclasses.dataclass()
class SnapshotBuilder:
"""Builds sdk snapshots"""
@@ -402,23 +407,35 @@ java_sdk_library_import {{
sdk_info_file_json = json.loads(sdk_info_file_object.read())
target_paths = []
target_dict = dict()
for jsonItem in sdk_info_file_json:
if not jsonItem["@type"] == "java_sdk_library":
continue
if not self.does_sdk_library_support_latest_api(jsonItem["@name"]):
sdk_library = jsonItem["@name"]
if not self.does_sdk_library_support_latest_api(sdk_library):
continue
target_dict[sdk_library] = dict()
for scope in jsonItem["scopes"]:
target_paths.append(jsonItem["scopes"][scope]["latest_api"])
target_paths.append(
jsonItem["scopes"][scope]["latest_removed_api"])
return target_paths
scope_json = jsonItem["scopes"][scope]
target_dict[sdk_library][scope] = dict()
target_list = [
"current_api", "latest_api", "removed_api",
"latest_removed_api"
]
for target in target_list:
target_dict[sdk_library][scope][target] = scope_json[target]
target_paths.append(scope_json["latest_api"])
target_paths.append(scope_json["latest_removed_api"])
return target_paths, target_dict
def build_sdk_scope_targets(self, build_release, sdk_version, modules):
# Build the latest scope targets for each module sdk
# Compute the paths to all the latest scope targets for each module sdk.
target_paths = []
target_dict = dict()
for module in modules:
for sdk in module.sdks:
if "host-exports" in sdk or "test-exports" in sdk:
@@ -426,8 +443,60 @@ java_sdk_library_import {{
sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir,
sdk, sdk_version)
target_paths.extend(self.latest_api_file_targets(sdk_info_file))
paths, dict_item = self.latest_api_file_targets(sdk_info_file)
target_paths.extend(paths)
target_dict[sdk_info_file] = dict_item
self.build_target_paths(build_release, sdk_version, target_paths)
return target_dict
def appendDiffToFile(self, file_object, sdk_zip_file, current_api,
latest_api, snapshots_dir):
"""Extract current api and find its diff with the latest api."""
with zipfile.ZipFile(sdk_zip_file, "r") as zipObj:
extracted_current_api = zipObj.extract(
member=current_api, path=snapshots_dir)
diff = subprocess.run(
["diff", "-u0", latest_api, extracted_current_api],
capture_output=True).stdout.decode("utf-8")
file_object.write(diff)
def create_snapshot_api_diff(self, sdk, sdk_version, target_dict,
snapshots_dir):
"""Creates api diff files for each module sdk.
For each module sdk, the scope targets are obtained for each java sdk
library and the api diff files are generated by performing a diff
operation between the current api file vs the latest api file.
"""
sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk, sdk_version)
sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk, sdk_version)
sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
sdk_version)
with open(sdk_api_diff_file, "w") as sdk_api_diff_file_object:
for sdk_library in target_dict[sdk_info_file]:
for scope in target_dict[sdk_info_file][sdk_library]:
scope_json = target_dict[sdk_info_file][sdk_library][scope]
current_api = scope_json["current_api"]
latest_api = scope_json["latest_api"]
self.appendDiffToFile(sdk_api_diff_file_object,
sdk_zip_file, current_api, latest_api,
snapshots_dir)
removed_api = scope_json["removed_api"]
latest_removed_api = scope_json["latest_removed_api"]
self.appendDiffToFile(sdk_api_diff_file_object,
sdk_zip_file, removed_api,
latest_removed_api, snapshots_dir)
def build_snapshot_api_diff(self, sdk_version, modules, target_dict,
snapshots_dir):
"""For each module sdk, create the api diff file."""
for module in modules:
for sdk in module.sdks:
if "host-exports" in sdk or "test-exports" in sdk:
continue
self.create_snapshot_api_diff(sdk, sdk_version, target_dict,
snapshots_dir)
# A list of the sdk versions to build. Usually just current but can include a
@@ -905,8 +974,10 @@ class SdkDistProducer:
snapshots_dir = self.snapshot_builder.build_snapshots(
build_release, sdk_versions, modules)
if build_release == LATEST:
self.snapshot_builder.build_sdk_scope_targets(
build_release, sdk_versions[0], modules)
target_dict = self.snapshot_builder.build_sdk_scope_targets(
build_release, "current", modules)
self.snapshot_builder.build_snapshot_api_diff(
"current", modules, target_dict, snapshots_dir)
self.populate_unbundled_dist(build_release, sdk_versions, modules,
snapshots_dir)
return snapshots_dir
@@ -919,6 +990,18 @@ class SdkDistProducer:
build_release, sdk_versions, modules)
self.populate_bundled_dist(build_release, modules, snapshots_dir)
def dist_sdk_snapshot_api_diff(self, sdk_dist_dir, sdk, sdk_version, module,
snapshots_dir):
"""Copy the sdk snapshot api diff file to a dist directory."""
if "host-exports" in sdk or "test-exports" in sdk:
return
sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, "sdk")
os.makedirs(sdk_dist_subdir, exist_ok=True)
sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
sdk_version)
shutil.copy(sdk_api_diff_path, sdk_dist_subdir)
def populate_unbundled_dist(self, build_release, sdk_versions, modules,
snapshots_dir):
build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
@@ -926,10 +1009,12 @@ class SdkDistProducer:
for module in modules:
for sdk_version in sdk_versions:
for sdk in module.sdks:
# TODO(b/230609867): create API diff file for each module
# sdk.
sdk_dist_dir = os.path.join(build_release_dist_dir,
sdk_version)
if build_release == LATEST:
self.dist_sdk_snapshot_api_diff(sdk_dist_dir, sdk,
sdk_version, module,
snapshots_dir)
self.populate_dist_snapshot(build_release, module, sdk,
sdk_dist_dir, sdk_version,
snapshots_dir)
@@ -964,7 +1049,7 @@ class SdkDistProducer:
and then a new zip file is created in the dist directory with the
original files replaced by the newly transformed files.
"""
os.makedirs(sdk_dist_dir)
os.makedirs(sdk_dist_dir, exist_ok=True)
dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip))
print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}")

36
build/mainline_modules_sdks_test.py
View File

@@ -47,7 +47,8 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
z.writestr(f"sdk_library/public/{name}-removed.txt", "")
z.writestr(f"sdk_library/public/{name}.srcjar", "")
z.writestr(f"sdk_library/public/{name}-stubs.jar", "")
z.writestr(f"sdk_library/public/{name}.txt", "")
z.writestr(f"sdk_library/public/{name}.txt",
"method public int testMethod(int);")
def create_snapshot_file(self, out_dir, name, version, for_r_build):
zip_file = Path(mm.sdk_snapshot_zip_file(out_dir, name, version))
@@ -74,7 +75,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
module.for_r_build)
return sdks_out_dir
def get_art_module_info_file_data(self):
def get_art_module_info_file_data(self, sdk):
info_file_data = f"""[
{{
"@type": "java_sdk_library",
@@ -85,10 +86,10 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
"dist_stem": "art",
"scopes": {{
"public": {{
"current_api": "sdk_library/public/art.module.public.api.txt",
"current_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}.txt",
"latest_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art.api.public.latest/gen/art.api.public.latest",
"latest_removed_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art-removed.api.public.latest/gen/art-removed.api.public.latest",
"removed_api": "sdk_library/public/art.module.public.api-removed.txt"
"removed_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}-removed.txt"
}}
}}
}}
@@ -101,16 +102,17 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
with open(file, "w") as file:
file.write(data)
def create_snapshot_info_file(self, module, sdk_info_file):
def create_snapshot_info_file(self, module, sdk_info_file, sdk):
if module == MAINLINE_MODULES_BY_APEX["com.android.art"]:
self.write_data_to_file(sdk_info_file,
self.get_art_module_info_file_data())
self.get_art_module_info_file_data(sdk))
else:
# For rest of the modules, generate an empty .info file.
self.write_data_to_file(sdk_info_file, "[]")
def build_sdk_scope_targets(self, build_release, sdk_version, modules):
target_paths = []
target_dict = dict()
for module in modules:
for sdk in module.sdks:
if "host-exports" in sdk or "test-exports" in sdk:
@@ -119,13 +121,16 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
sdk_info_file = mm.sdk_snapshot_info_file(
Path(self.mainline_sdks_dir).joinpath("test"), sdk,
sdk_version)
self.create_snapshot_info_file(module, sdk_info_file)
target_paths.extend(self.latest_api_file_targets(sdk_info_file))
self.create_snapshot_info_file(module, sdk_info_file, sdk)
paths, dict_item = self.latest_api_file_targets(sdk_info_file)
target_paths.extend(paths)
target_dict[sdk_info_file] = dict_item
for target_path in target_paths:
os.makedirs(os.path.split(target_path)[0])
self.write_data_to_file(target_path, "")
# TODO(b/230609867): Add test to verify api diff file generation.
return target_dict
class TestProduceDist(unittest.TestCase):
@@ -195,9 +200,12 @@ class TestProduceDist(unittest.TestCase):
"mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-S-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
],
sorted(self.list_files_in_dir(self.tmp_dist_dir)))
@@ -273,11 +281,21 @@ class TestProduceDist(unittest.TestCase):
"bundled-mainline-sdks/platform-mainline/test-exports/platform-mainline-test-exports-current.zip",
# Unbundled (normal) modules.
"mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
"mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
],
sorted(self.list_files_in_dir(self.tmp_dist_dir)))
art_api_diff_file = os.path.join(
self.tmp_dist_dir,
"mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt"
)
self.assertNotEqual(
os.path.getsize(art_api_diff_file),
0,
msg="Api diff file should not be empty for the art module")
def create_build_number_file(self):
soong_dir = os.path.join(self.tmp_out_dir, "soong")
os.makedirs(soong_dir, exist_ok=True)