Merge "Revert "Revert "Add tools for updating NDK ABI dumps.""" am: dff4dc6240

Original change: https://android-review.googlesource.com/c/platform/development/+/2336135

Change-Id: I56790789dee3509260e9444536a4e34e24c3f95b
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

tools/ndk/Android.bp

@@ -0,0 +1,27 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+python_binary_host {
+    name: "update-ndk-abi",
+    pkg_path: "update_ndk_abi",
+    main: "update_ndk_abi.py",
+    srcs: [
+        "update_ndk_abi.py",
+    ],
+    libs: [
+        "ndkabidump"
+    ],
+}

tools/ndk/mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+disallow_untyped_defs = True

tools/ndk/ndkabidump/Android.bp

@@ -0,0 +1,25 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+python_library_host {
+    name: "ndkabidump",
+    pkg_path: "ndkabidump",
+    srcs: [
+        "__init__.py",
+        "soong.py",
+    ],
+}
+

tools/ndk/ndkabidump/__init__.py

@@ -0,0 +1,132 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tool for updating the prebuilt NDK ABI dumps."""
+import argparse
+import logging
+from pathlib import Path
+import shutil
+import sys
+
+from .soong import Soong
+
+
+def logger() -> logging.Logger:
+    """Returns the module level logger."""
+    return logging.getLogger(__name__)
+
+
+class Updater:
+    """Tool for updating prebuilt NDK ABI dumps."""
+
+    def __init__(self, src_dir: Path, build_dir: Path) -> None:
+        self.src_dir = src_dir
+        self.build_dir = build_dir
+
+    def build_abi_dumps(self) -> None:
+        """Builds the updated NDK ABI dumps."""
+        soong = Soong(self.src_dir, self.build_dir)
+        logger().info(f"Building ABI dumps to {self.build_dir}")
+        soong.build(["dump-ndk-abi"], env={"TARGET_PRODUCT": "ndk"})
+
+    def copy_updated_abi_dumps(self) -> None:
+        """Copies the NDK ABI dumps from the build directory to prebuilts."""
+        prebuilts_project = self.src_dir / "prebuilts/abi-dumps"
+        prebuilts_dir = prebuilts_project / "ndk"
+        abi_out = self.build_dir / "soong/abi-dumps/ndk"
+        for dump in abi_out.glob("**/abi.xml"):
+            install_path = prebuilts_dir / dump.relative_to(abi_out)
+            install_dir = install_path.parent
+            if not install_dir.exists():
+                install_dir.mkdir(parents=True)
+            logger().info(f"Copying ABI dump {dump} to {install_path}")
+            shutil.copy2(dump, install_path)
+
+    def run(self) -> None:
+        """Runs the updater.
+
+        Cleans the out directory, builds the ABI dumps, and copies the results
+        to the prebuilts directory.
+        """
+        self.build_abi_dumps()
+        self.copy_updated_abi_dumps()
+
+
+HELP = """\
+Builds and updates the NDK ABI prebuilts.
+
+Whenever a change is made that alters the NDK ABI (or an API level is
+finalized, or a new preview codename is introduced to the build), the prebuilts
+in prebuilts/abi-dumps/ndk need to be updated to match. For any finalized APIs,
+the breaking change typically needs to be reverted.
+
+Note that typically this tool should be executed via
+development/tools/ndk/update_ndk_abi.sh. That script will ensure that this tool
+is up-to-date and run with the correct arguments.
+"""
+
+
+class App:
+    """Command line application from updating NDK ABI prebuilts."""
+
+    @staticmethod
+    def parse_args() -> argparse.Namespace:
+        """Parses and returns command line arguments."""
+
+        parser = argparse.ArgumentParser(
+            formatter_class=argparse.RawDescriptionHelpFormatter, description=HELP
+        )
+
+        def resolved_path(path: str) -> Path:
+            """Converts a string into a fully resolved Path."""
+            return Path(path).resolve()
+
+        parser.add_argument(
+            "--src-dir",
+            type=resolved_path,
+            required=True,
+            help="Path to the top of the Android source tree.",
+        )
+
+        parser.add_argument(
+            "out_dir",
+            type=resolved_path,
+            metavar="OUT_DIR",
+            help="Output directory to use for building ABI dumps.",
+        )
+
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=0,
+            help="Increase logging verbosity.",
+        )
+
+        return parser.parse_args()
+
+    def run(self) -> None:
+        """Builds the new NDK ABI dumps and copies them to prebuilts."""
+        args = self.parse_args()
+        log_level = logging.DEBUG if args.verbose else logging.INFO
+        logging.basicConfig(level=log_level)
+        test_path = args.src_dir / "build/soong/soong_ui.bash"
+        if not test_path.exists():
+            sys.exit(
+                f"Source directory {args.src_dir} does not appear to be an "
+                f"Android source tree: {test_path} does not exist."
+            )
+
+        Updater(args.src_dir, args.out_dir).run()

tools/ndk/ndkabidump/soong.py

@@ -0,0 +1,112 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""APIs for interacting with Soong."""
+import logging
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+
+
+def logger() -> logging.Logger:
+    """Returns the module level logger."""
+    return logging.getLogger(__name__)
+
+
+class Soong:
+    """Interface for interacting with Soong."""
+
+    def __init__(self, build_top: Path, out_dir: Path) -> None:
+        self.out_dir = out_dir
+        self.soong_ui_path = build_top / "build/soong/soong_ui.bash"
+
+    def soong_ui(
+        self,
+        args: list[str],
+        env: dict[str, str] | None = None,
+        capture_output: bool = False,
+    ) -> str:
+        """Executes soong_ui.bash and returns the output on success.
+
+        Args:
+            args: List of string arguments to pass to soong_ui.bash.
+            env: Additional environment variables to set when running soong_ui.
+            capture_output: True if the output of the command should be captured and
+                returned. If not, the output will be printed to stdout/stderr and an
+                empty string will be returned.
+
+        Raises:
+            subprocess.CalledProcessError: The subprocess failure if the soong command
+            failed.
+
+        Returns:
+            The interleaved contents of stdout/stderr if capture_output is True, else an
+            empty string.
+        """
+        if env is None:
+            env = {}
+
+        # Use a (mostly) clean environment to avoid the caller's lunch
+        # environment affecting the build.
+        exec_env = {
+            # Newer versions of golang require the go cache, which defaults to somewhere
+            # in HOME if not set.
+            "HOME": os.environ["HOME"],
+            "OUT_DIR": str(self.out_dir.resolve()),
+            "PATH": os.environ["PATH"],
+        }
+        exec_env.update(env)
+        env_prefix = " ".join(f"{k}={v}" for k, v in exec_env.items())
+        cmd = [str(self.soong_ui_path)] + args
+        logger().debug(f"running in {os.getcwd()}: {env_prefix} {shlex.join(cmd)}")
+        result = subprocess.run(
+            cmd,
+            check=True,
+            capture_output=capture_output,
+            encoding="utf-8",
+            env=exec_env,
+        )
+        return result.stdout
+
+    def get_make_var(self, name: str) -> str:
+        """Queries the build system for the value of a make variable.
+
+        Args:
+            name: The name of the build variable to query.
+
+        Returns:
+            The value of the build variable in string form.
+        """
+        return self.soong_ui(["--dumpvar-mode", name], capture_output=True).rstrip("\n")
+
+    def clean(self) -> None:
+        """Removes the output directory, if it exists."""
+        if self.out_dir.exists():
+            shutil.rmtree(self.out_dir)
+
+    def build(self, targets: list[str], env: dict[str, str] | None = None) -> None:
+        """Builds the given targets.
+
+        The out directory will be created if it does not already exist. Existing
+        contents will not be removed, but affected outputs will be modified.
+
+        Args:
+            targets: A list of target names to build.
+            env: Additional environment variables to set when running the build.
+        """
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self.soong_ui(["--make-mode", "--soong-only"] + targets, env=env)

tools/ndk/ndkabidump/test_soong.py

@@ -0,0 +1,93 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tests for Soong APIs."""
+import contextlib
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import unittest.mock
+
+from .soong import Soong
+
+
+if "ANDROID_BUILD_TOP" not in os.environ:
+    raise RuntimeError(
+        "Cannot run Soong tests without ANDROID_BUILD_TOP defined. Run lunch."
+    )
+
+
+ANDROID_BUILD_TOP = Path(os.environ["ANDROID_BUILD_TOP"]).resolve()
+
+
+class SoongTest(unittest.TestCase):
+    """Tests for the Soong executor."""
+
+    out_dir: Path
+
+    def setUp(self) -> None:
+        with contextlib.ExitStack() as stack:
+            self.out_dir = Path(stack.enter_context(tempfile.TemporaryDirectory()))
+            self.addCleanup(stack.pop_all().close)
+
+    def test_finds_soong_ui(self) -> None:
+        """Tests that soong_ui.bash is found correctly."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        self.assertTrue(soong.soong_ui_path.exists())
+        self.assertEqual("soong_ui.bash", soong.soong_ui_path.name)
+
+    def test_get_build_var(self) -> None:
+        """Tests that we can read build variables from Soong."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        old_product = os.environ["TARGET_PRODUCT"]
+        try:
+            # Clear the lunched target out of the test environment for a
+            # consistent result.
+            del os.environ["TARGET_PRODUCT"]
+            self.assertEqual("generic", soong.get_make_var("TARGET_DEVICE"))
+        finally:
+            os.environ["TARGET_PRODUCT"] = old_product
+
+    def test_clean(self) -> None:
+        """Tests that clean works."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        self.assertTrue(self.out_dir.exists())
+        soong.clean()
+        self.assertFalse(self.out_dir.exists())
+        soong.clean()
+        self.assertFalse(self.out_dir.exists())
+
+    def test_build(self) -> None:
+        """Tests that build invokes the correct command.
+
+        Does not actually test a build, as there aren't any good options for short
+        builds in the tree, so instead tests that soong_ui is called the way we expect
+        it to be.
+        """
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        with unittest.mock.patch.object(soong, "soong_ui") as soong_ui:
+            soong.build(["foo"])
+            soong_ui.assert_called_with(
+                ["--make-mode", "--soong-only", "foo"], env=None
+            )
+
+    def test_build_creates_out_dir(self) -> None:
+        """Tests that build creates the out directory if necessary."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        soong.clean()
+        with unittest.mock.patch.object(soong, "soong_ui"):
+            soong.build([])
+        self.assertTrue(self.out_dir.exists())

tools/ndk/pylintrc

@@ -0,0 +1,2 @@
+[MESSAGES CONTROL]
+disable=logging-fstring-interpolation

tools/ndk/update_ndk_abi.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Entry point for NDK ABI prebuilt updater."""
+from ndkabidump import App
+
+
+if __name__ == "__main__":
+    App().run()

tools/ndk/update_ndk_abi.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -z "$ANDROID_BUILD_TOP" ]]; then
+  >&2 echo "ANDROID_BUILD_TOP not set in environment. Run lunch."
+  exit 1
+fi
+
+set -e
+set -x
+
+$ANDROID_BUILD_TOP/build/soong/soong_ui.bash --make-mode update-ndk-abi
+update-ndk-abi --src-dir $ANDROID_BUILD_TOP $ANDROID_BUILD_TOP/ndk-abi-out "$@"