Merge "Generating api diff files for each module sdk."

build/mainline_modules_sdks.py

@@ -250,6 +250,11 @@ def sdk_snapshot_info_file(snapshots_dir, sdk_name, sdk_version):
     return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}.info")
 
 
+def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name, sdk_version):
+    """Get the path to the sdk snapshot api diff file."""
+    return os.path.join(snapshots_dir, f"{sdk_name}-{sdk_version}-api-diff.txt")
+
+
 @dataclasses.dataclass()
 class SnapshotBuilder:
     """Builds sdk snapshots"""
@@ -425,23 +430,35 @@ java_sdk_library_import {{
             sdk_info_file_json = json.loads(sdk_info_file_object.read())
 
         target_paths = []
+        target_dict = dict()
         for jsonItem in sdk_info_file_json:
             if not jsonItem["@type"] == "java_sdk_library":
                 continue
 
-            if not self.does_sdk_library_support_latest_api(jsonItem["@name"]):
+            sdk_library = jsonItem["@name"]
+            if not self.does_sdk_library_support_latest_api(sdk_library):
                 continue
 
+            target_dict[sdk_library] = dict()
             for scope in jsonItem["scopes"]:
-                target_paths.append(jsonItem["scopes"][scope]["latest_api"])
+                scope_json = jsonItem["scopes"][scope]
-                target_paths.append(
+                target_dict[sdk_library][scope] = dict()
-                    jsonItem["scopes"][scope]["latest_removed_api"])
+                target_list = [
-        return target_paths
+                    "current_api", "latest_api", "removed_api",
+                    "latest_removed_api"
+                ]
+                for target in target_list:
+                    target_dict[sdk_library][scope][target] = scope_json[target]
+                target_paths.append(scope_json["latest_api"])
+                target_paths.append(scope_json["latest_removed_api"])
+
+        return target_paths, target_dict
 
     def build_sdk_scope_targets(self, build_release, sdk_version, modules):
         # Build the latest scope targets for each module sdk
         # Compute the paths to all the latest scope targets for each module sdk.
         target_paths = []
+        target_dict = dict()
         for module in modules:
             for sdk in module.sdks:
                 if "host-exports" in sdk or "test-exports" in sdk:
@@ -449,8 +466,60 @@ java_sdk_library_import {{
 
                 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir,
                                                        sdk, sdk_version)
-                target_paths.extend(self.latest_api_file_targets(sdk_info_file))
+                paths, dict_item = self.latest_api_file_targets(sdk_info_file)
+                target_paths.extend(paths)
+                target_dict[sdk_info_file] = dict_item
         self.build_target_paths(build_release, sdk_version, target_paths)
+        return target_dict
+
+    def appendDiffToFile(self, file_object, sdk_zip_file, current_api,
+                         latest_api, snapshots_dir):
+        """Extract current api and find its diff with the latest api."""
+        with zipfile.ZipFile(sdk_zip_file, "r") as zipObj:
+            extracted_current_api = zipObj.extract(
+                member=current_api, path=snapshots_dir)
+            diff = subprocess.run(
+                ["diff", "-u0", latest_api, extracted_current_api],
+                capture_output=True).stdout.decode("utf-8")
+            file_object.write(diff)
+
+    def create_snapshot_api_diff(self, sdk, sdk_version, target_dict,
+                                 snapshots_dir):
+        """Creates api diff files for each module sdk.
+
+        For each module sdk, the scope targets are obtained for each java sdk
+        library and the api diff files are generated by performing a diff
+        operation between the current api file vs the latest api file.
+        """
+        sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk, sdk_version)
+        sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk, sdk_version)
+        sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
+                                                       sdk_version)
+        with open(sdk_api_diff_file, "w") as sdk_api_diff_file_object:
+            for sdk_library in target_dict[sdk_info_file]:
+                for scope in target_dict[sdk_info_file][sdk_library]:
+                    scope_json = target_dict[sdk_info_file][sdk_library][scope]
+                    current_api = scope_json["current_api"]
+                    latest_api = scope_json["latest_api"]
+                    self.appendDiffToFile(sdk_api_diff_file_object,
+                                          sdk_zip_file, current_api, latest_api,
+                                          snapshots_dir)
+
+                    removed_api = scope_json["removed_api"]
+                    latest_removed_api = scope_json["latest_removed_api"]
+                    self.appendDiffToFile(sdk_api_diff_file_object,
+                                          sdk_zip_file, removed_api,
+                                          latest_removed_api, snapshots_dir)
+
+    def build_snapshot_api_diff(self, sdk_version, modules, target_dict,
+                                snapshots_dir):
+        """For each module sdk, create the api diff file."""
+        for module in modules:
+            for sdk in module.sdks:
+                if "host-exports" in sdk or "test-exports" in sdk:
+                    continue
+                self.create_snapshot_api_diff(sdk, sdk_version, target_dict,
+                                              snapshots_dir)
 
 
 # A list of the sdk versions to build. Usually just current but can include a
@@ -976,8 +1045,10 @@ class SdkDistProducer:
         snapshots_dir = self.snapshot_builder.build_snapshots(
             build_release, sdk_versions, modules)
         if build_release == LATEST:
-            self.snapshot_builder.build_sdk_scope_targets(
+            target_dict = self.snapshot_builder.build_sdk_scope_targets(
-                build_release, sdk_versions[0], modules)
+                build_release, "current", modules)
+            self.snapshot_builder.build_snapshot_api_diff(
+                "current", modules, target_dict, snapshots_dir)
         self.populate_unbundled_dist(build_release, sdk_versions, modules,
                                      snapshots_dir)
         return snapshots_dir
@@ -990,6 +1061,18 @@ class SdkDistProducer:
                 build_release, sdk_versions, modules)
             self.populate_bundled_dist(build_release, modules, snapshots_dir)
 
+    def dist_sdk_snapshot_api_diff(self, sdk_dist_dir, sdk, sdk_version, module,
+                                   snapshots_dir):
+        """Copy the sdk snapshot api diff file to a dist directory."""
+        if "host-exports" in sdk or "test-exports" in sdk:
+            return
+
+        sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, "sdk")
+        os.makedirs(sdk_dist_subdir, exist_ok=True)
+        sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk,
+                                                       sdk_version)
+        shutil.copy(sdk_api_diff_path, sdk_dist_subdir)
+
     def populate_unbundled_dist(self, build_release, sdk_versions, modules,
                                 snapshots_dir):
         build_release_dist_dir = os.path.join(self.mainline_sdks_dir,
@@ -997,10 +1080,12 @@ class SdkDistProducer:
         for module in modules:
             for sdk_version in sdk_versions:
                 for sdk in module.sdks:
-                    # TODO(b/230609867): create API diff file for each module
-                    # sdk.
                     sdk_dist_dir = os.path.join(build_release_dist_dir,
                                                 sdk_version)
+                    if build_release == LATEST:
+                        self.dist_sdk_snapshot_api_diff(sdk_dist_dir, sdk,
+                                                        sdk_version, module,
+                                                        snapshots_dir)
                     self.populate_dist_snapshot(build_release, module, sdk,
                                                 sdk_dist_dir, sdk_version,
                                                 snapshots_dir)
@@ -1035,7 +1120,7 @@ class SdkDistProducer:
         and then a new zip file is created in the dist directory with the
         original files replaced by the newly transformed files.
         """
-        os.makedirs(sdk_dist_dir)
+        os.makedirs(sdk_dist_dir, exist_ok=True)
         dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip))
         print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}")
 

build/mainline_modules_sdks_test.py

@@ -47,7 +47,8 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
         z.writestr(f"sdk_library/public/{name}-removed.txt", "")
         z.writestr(f"sdk_library/public/{name}.srcjar", "")
         z.writestr(f"sdk_library/public/{name}-stubs.jar", "")
-        z.writestr(f"sdk_library/public/{name}.txt", "")
+        z.writestr(f"sdk_library/public/{name}.txt",
+                   "method public int testMethod(int);")
 
     def create_snapshot_file(self, out_dir, name, version, for_r_build):
         zip_file = Path(mm.sdk_snapshot_zip_file(out_dir, name, version))
@@ -74,7 +75,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
                                               module.for_r_build)
         return sdks_out_dir
 
-    def get_art_module_info_file_data(self):
+    def get_art_module_info_file_data(self, sdk):
         info_file_data = f"""[
   {{
     "@type": "java_sdk_library",
@@ -85,10 +86,10 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
     "dist_stem": "art",
     "scopes": {{
       "public": {{
-        "current_api": "sdk_library/public/art.module.public.api.txt",
+        "current_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}.txt",
         "latest_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art.api.public.latest/gen/art.api.public.latest",
         "latest_removed_api": "{Path(self.mainline_sdks_dir).joinpath("test")}/prebuilts/sdk/art-removed.api.public.latest/gen/art-removed.api.public.latest",
-        "removed_api": "sdk_library/public/art.module.public.api-removed.txt"
+        "removed_api": "sdk_library/public/{re.sub(r"-.*$", "", sdk)}-removed.txt"
       }}
     }}
   }}
@@ -101,16 +102,17 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
         with open(file, "w") as file:
             file.write(data)
 
-    def create_snapshot_info_file(self, module, sdk_info_file):
+    def create_snapshot_info_file(self, module, sdk_info_file, sdk):
         if module == MAINLINE_MODULES_BY_APEX["com.android.art"]:
             self.write_data_to_file(sdk_info_file,
-                                    self.get_art_module_info_file_data())
+                                    self.get_art_module_info_file_data(sdk))
         else:
             # For rest of the modules, generate an empty .info file.
             self.write_data_to_file(sdk_info_file, "[]")
 
     def build_sdk_scope_targets(self, build_release, sdk_version, modules):
         target_paths = []
+        target_dict = dict()
         for module in modules:
             for sdk in module.sdks:
                 if "host-exports" in sdk or "test-exports" in sdk:
@@ -119,13 +121,16 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder):
                 sdk_info_file = mm.sdk_snapshot_info_file(
                     Path(self.mainline_sdks_dir).joinpath("test"), sdk,
                     sdk_version)
-                self.create_snapshot_info_file(module, sdk_info_file)
+                self.create_snapshot_info_file(module, sdk_info_file, sdk)
-                target_paths.extend(self.latest_api_file_targets(sdk_info_file))
+                paths, dict_item = self.latest_api_file_targets(sdk_info_file)
+                target_paths.extend(paths)
+                target_dict[sdk_info_file] = dict_item
 
         for target_path in target_paths:
             os.makedirs(os.path.split(target_path)[0])
             self.write_data_to_file(target_path, "")
-            # TODO(b/230609867): Add test to verify api diff file generation.
+
+        return target_dict
 
 
 class TestProduceDist(unittest.TestCase):
@@ -195,9 +200,12 @@ class TestProduceDist(unittest.TestCase):
                 "mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
                 "mainline-sdks/for-S-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip",
+                "mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.google.android.wifi/sdk/wifi-module-sdk-current.zip",
             ],
             sorted(self.list_files_in_dir(self.tmp_dist_dir)))
@@ -273,11 +281,21 @@ class TestProduceDist(unittest.TestCase):
                 "bundled-mainline-sdks/platform-mainline/test-exports/platform-mainline-test-exports-current.zip",
                 # Unbundled (normal) modules.
                 "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip",
+                "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt",
                 "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip",
                 "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip",
             ],
             sorted(self.list_files_in_dir(self.tmp_dist_dir)))
 
+        art_api_diff_file = os.path.join(
+            self.tmp_dist_dir,
+            "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current-api-diff.txt"
+        )
+        self.assertNotEqual(
+            os.path.getsize(art_api_diff_file),
+            0,
+            msg="Api diff file should not be empty for the art module")
+
     def create_build_number_file(self):
         soong_dir = os.path.join(self.tmp_out_dir, "soong")
         os.makedirs(soong_dir, exist_ok=True)