Generating api diff files for each module sdk.

In order to detect the addition of any new APIs in any mainline module,
an API diff file (current module sdk vs the last finalized module sdk)
needs to be generated correpsonding to each module sdk.

This CL generates the API diff files corresponding to each module sdk
and copy that to the out/dist directory.

This CL also contains the tests to verify the generation of api diff
files.

Bug: 230609867
Test: builds successfully. Generates API diff files for each module sdk
"atest mainline_modules_sdks_test" passes.

Change-Id: I1d8276a4054b92490ccca3dae36c48c639925f46

build/mainline_modules_sdks.py

@@ -227,6 +227,11 @@ def sdk_snapshot_info_file(snapshots_dir, sdk_name, sdk_version):
     return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}.info")
 
 
+def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name, sdk_version):
+    """Get the path to the sdk snapshot api diff file."""
+    return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}-api-diff.txt")
+
+
 @dataclasses.dataclass()
 class SnapshotBuilder:
     """Builds sdk snapshots"""
@@ -402,23 +407,35 @@ java_sdk_library_import {{
             sdk_info_file_json = json.loads(sdk_info_file_object.read())
 
         target_paths = []
+        target_dict = dict()
         for jsonItem in sdk_info_file_json:
             if not jsonItem["@type"] == "java_sdk_library":
                 continue
 
-            if not self.does_sdk_library_support_latest_api(jsonItem["@name"]):
+            sdk_library = jsonItem["@name"]
+            if not self.does_sdk_library_support_latest_api(sdk_library):
                 continue
 
+            target_dict[sdk_library] = dict()
             for scope in jsonItem["scopes"]:
-                target_paths.append(jsonItem["scopes"][scope]["latest_api"])
+                scope_json = jsonItem["scopes"][scope]
-                target_paths.append(
+                target_dict[sdk_library][scope] = dict()
-                    jsonItem["scopes"][scope]["latest_removed_api"])
+                target_list = [
-        return target_paths
+                    "current_api", "latest_api", "removed_api",
+                    "latest_removed_api"
+                ]
+                for target in target_list:
+                    target_dict[sdk_library][scope][target] = scope_json[target]
+                target_paths.append(scope_json["latest_api"])
+                target_paths.append(scope_json["latest_removed_api"])
+
+        return target_paths, target_dict
 
     def build_sdk_scope_targets(self, build_release, sdk_version, modules):
         # Build the latest scope targets for each module sdk
         # Compute the paths to all the latest scope targets for each module sdk.
         target_paths = []
+        target_dict = dict()
         for module in modules:
             for sdk in module.sdks:
                 if "host-exports" in sdk or "test-exports" in sdk:
@@ -426,8 +443,60 @@ java_sdk_library_import {{
 
                 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir,
                                                        sdk, sdk_version)
-                target_paths.extend(self.latest_api_file_targets(sdk_info_file))
+                paths, dict_item = self.latest_api_file_targets(sdk_info_file)
+                target_paths.extend(paths)
+                target_dict[sdk_info_file] = dict_item
         self.build_target_paths(build_release, sdk_version, target_paths)
+        return target_dict
+
+    def appendDiffToFile(self, file_object, sdk_zip_file, current_api,
+                         latest_api, snapshots_dir):
+        """Extract current api and find its diff with the latest api."""
+        with zipfile.ZipFile(sdk_zip_file, "r") as zipObj:
+            extracted_current_api = zipObj.extract(
+                member=current_api, path=snapshots_dir)
+            diff = subprocess.run(
+                ["diff", "-u0", latest_api, extracted_current_api],
+                capture_output=True).stdout.decode("utf-8")
+            file_object.write(diff)
+
+    def create_snapshot_api_diff(self, sdk, sdk_version, target_dict,
+                                 snapshots_dir):
+        """Creates api diff files for each module sdk.
+
+        For each module sdk, the scope targets are obtained for each java sdk
+        library and the api diff files are generated by performing a diff
+        operation between the current api file vs the latest api file.
+        """
+        sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk, sdk_version)
+        sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk, sdk_version)
+        sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
+                                                       sdk_version)
+        with open(sdk_api_diff_file, "w") as sdk_api_diff_file_object:
+            for sdk_library in target_dict[sdk_info_file]:
+                for scope in target_dict[sdk_info_file][sdk_library]:
+                    scope_json = target_dict[sdk_info_file][sdk_library][scope]
+                    current_api = scope_json["current_api"]
+                    latest_api = scope_json["latest_api"]
+                    self.appendDiffToFile(sdk_api_diff_file_object,
+                                          sdk_zip_file, current_api, latest_api,
+                                          snapshots_dir)
+
+                    removed_api = scope_json["removed_api"]
+                    latest_removed_api = scope_json["latest_removed_api"]
+                    self.appendDiffToFile(sdk_api_diff_file_object,
+                                          sdk_zip_file, removed_api,
+                                          latest_removed_api, snapshots_dir)
+
+    def build_snapshot_api_diff(self, sdk_version, modules, target_dict,
+                                snapshots_dir):
+        """For each module sdk, create the api diff file."""
+        for module in modules:
+            for sdk in module.sdks:
+                if "host-exports" in sdk or "test-exports" in sdk:
+                    continue
+                self.create_snapshot_api_diff(sdk, sdk_version, target_dict,
+                                              snapshots_dir)
 
 
 # A list of the sdk versions to build. Usually just current but can include a
@@ -905,8 +974,10 @@ class SdkDistProducer:
         snapshots_dir = self.snapshot_builder.build_snapshots(
             build_release, sdk_versions, modules)
         if build_release == LATEST:
-            self.snapshot_builder.build_sdk_scope_targets(
+            target_dict = self.snapshot_builder.build_sdk_scope_targets(
-                build_release, sdk_versions[0], modules)
+                build_release, "current", modules)
+            self.snapshot_builder.build_snapshot_api_diff(
+                "current", modules, target_dict, snapshots_dir)
         self.populate_unbundled_dist(build_release, sdk_versions, modules,
                                      snapshots_dir)
         return snapshots_dir
@@ -919,6 +990,18 @@ class SdkDistProducer:
                 build_release, sdk_versions, modules)
             self.populate_bundled_dist(build_release, modules, snapshots_dir)
 
+    def dist_sdk_snapshot_api_diff(self, sdk_dist_dir, sdk, sdk_version, module,
+                                   snapshots_dir):
+        """Copy the sdk snapshot api diff file to a dist directory."""
+        if "host-exports" in sdk or "test-exports" in sdk:
+            return
+
+        sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, "sdk")
+        os.makedirs(sdk_dist_subdir, exist_ok=True)
+        sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
+                                                       sdk_version)
+        shutil.copy(sdk_api_diff_path, sdk_dist_subdir)
+
     def populate_unbundled_dist(self, build_release, sdk_versions, modules,
                                 snapshots_dir):
         build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
@@ -926,10 +1009,12 @@ class SdkDistProducer:
         for module in modules:
             for sdk_version in sdk_versions:
                 for sdk in module.sdks:
-                    # TODO(b/230609867): create API diff file for each module
-                    # sdk.
                     sdk_dist_dir = os.path.join(build_release_dist_dir,
                                                 sdk_version)
+                    if build_release == LATEST:
+                        self.dist_sdk_snapshot_api_diff(sdk_dist_dir, sdk,
+                                                        sdk_version, module,
+                                                        snapshots_dir)
                     self.populate_dist_snapshot(build_release, module, sdk,
                                                 sdk_dist_dir, sdk_version,
                                                 snapshots_dir)
@@ -964,7 +1049,7 @@ class SdkDistProducer:
         and then a new zip file is created in the dist directory with the
         original files replaced by the newly transformed files.
         """
-        os.makedirs(sdk_dist_dir)
+        os.makedirs(sdk_dist_dir, exist_ok=True)
         dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip))
         print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}")
 

build/mainline_modules_sdks_test.py

@@ -47,7 +47,8 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
         z.writestr(f"sdk_library/public/{name}-removed.txt", "")
         z.writestr(f"sdk_library/public/{name}.srcjar", "")
         z.writestr(f"sdk_library/public/{name}-stubs.jar", "")
-        z.writestr(f"sdk_library/public/{name}.txt", "")
+        z.writestr(f"sdk_library/public/{name}.txt",
+                   "method public int testMethod(int);")
 
     def create_snapshot_file(self, out_dir, name, version, for_r_build):
         zip_file = Path(mm.sdk_snapshot_zip_file(out_dir, name, version))
@@ -74,7 +75,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
                                               module.for_r_build)
         return sdks_out_dir
 
-    def get_art_module_info_file_data(self):
+    def get_art_module_info_file_data(self, sdk):
         info_file_data = f"""[
   {{
     "@type": "java_sdk_library",
@@ -85,10 +86,10 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
     "dist_stem": "art",
     "scopes": {{
       "public": {{
-        "current_api": "sdk_library/public/art.module.public.api.txt",
+        "current_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}.txt",
         "latest_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art.api.public.latest/gen/art.api.public.latest",
         "latest_removed_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art-removed.api.public.latest/gen/art-removed.api.public.latest",
-        "removed_api": "sdk_library/public/art.module.public.api-removed.txt"
+        "removed_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}-removed.txt"
       }}
     }}
   }}
@@ -101,16 +102,17 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
         with open(file, "w") as file:
             file.write(data)
 
-    def create_snapshot_info_file(self, module, sdk_info_file):
+    def create_snapshot_info_file(self, module, sdk_info_file, sdk):
         if module == MAINLINE_MODULES_BY_APEX["com.android.art"]:
             self.write_data_to_file(sdk_info_file,
-                                    self.get_art_module_info_file_data())
+                                    self.get_art_module_info_file_data(sdk))
         else:
             # For rest of the modules, generate an empty .info file.
             self.write_data_to_file(sdk_info_file, "[]")
 
     def build_sdk_scope_targets(self, build_release, sdk_version, modules):
         target_paths = []
+        target_dict = dict()
         for module in modules:
             for sdk in module.sdks:
                 if "host-exports" in sdk or "test-exports" in sdk:
@@ -119,13 +121,16 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
                 sdk_info_file = mm.sdk_snapshot_info_file(
                     Path(self.mainline_sdks_dir).joinpath("test"), sdk,
                     sdk_version)
-                self.create_snapshot_info_file(module, sdk_info_file)
+                self.create_snapshot_info_file(module, sdk_info_file, sdk)
-                target_paths.extend(self.latest_api_file_targets(sdk_info_file))
+                paths, dict_item = self.latest_api_file_targets(sdk_info_file)
+                target_paths.extend(paths)
+                target_dict[sdk_info_file] = dict_item
 
         for target_path in target_paths:
             os.makedirs(os.path.split(target_path)[0])
             self.write_data_to_file(target_path, "")
-            # TODO(b/230609867): Add test to verify api diff file generation.
+
+        return target_dict
 
 
 class TestProduceDist(unittest.TestCase):
@@ -195,9 +200,12 @@ class TestProduceDist(unittest.TestCase):
                 "mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
                 "mainline-sdks/for-S-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
+                "mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
             ],
             sorted(self.list_files_in_dir(self.tmp_dist_dir)))
@@ -273,11 +281,21 @@ class TestProduceDist(unittest.TestCase):
                 "bundled-mainline-sdks/platform-mainline/test-exports/platform-mainline-test-exports-current.zip",
                 # Unbundled (normal) modules.
                 "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
             ],
             sorted(self.list_files_in_dir(self.tmp_dist_dir)))
 
+        art_api_diff_file = os.path.join(
+            self.tmp_dist_dir,
+            "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt"
+        )
+        self.assertNotEqual(
+            os.path.getsize(art_api_diff_file),
+            0,
+            msg="Api diff file should not be empty for the art module")
+
     def create_build_number_file(self):
         soong_dir = os.path.join(self.tmp_out_dir, "soong")
         os.makedirs(soong_dir, exist_ok=True)