Revert "Revert "Add tools for updating NDK ABI dumps.""

Can't figure out how to get the tests to work when built by soong, so
just removing for now.

This reverts commit d97284de03.

Test: that was the problem
Change-Id: I4bb8c3a81f529fabbc1663b48f93f153db23aa5d

tools/ndk/Android.bp

@@ -0,0 +1,27 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+python_binary_host {
+    name: "update-ndk-abi",
+    pkg_path: "update_ndk_abi",
+    main: "update_ndk_abi.py",
+    srcs: [
+        "update_ndk_abi.py",
+    ],
+    libs: [
+        "ndkabidump"
+    ],
+}

tools/ndk/mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+disallow_untyped_defs = True

tools/ndk/ndkabidump/Android.bp

@@ -0,0 +1,25 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+python_library_host {
+    name: "ndkabidump",
+    pkg_path: "ndkabidump",
+    srcs: [
+        "__init__.py",
+        "soong.py",
+    ],
+}
+

tools/ndk/ndkabidump/__init__.py

@@ -0,0 +1,132 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tool for updating the prebuilt NDK ABI dumps."""
+import argparse
+import logging
+from pathlib import Path
+import shutil
+import sys
+
+from .soong import Soong
+
+
+def logger() -> logging.Logger:
+    """Returns the module level logger."""
+    return logging.getLogger(__name__)
+
+
+class Updater:
+    """Tool for updating prebuilt NDK ABI dumps."""
+
+    def __init__(self, src_dir: Path, build_dir: Path) -> None:
+        self.src_dir = src_dir
+        self.build_dir = build_dir
+
+    def build_abi_dumps(self) -> None:
+        """Builds the updated NDK ABI dumps."""
+        soong = Soong(self.src_dir, self.build_dir)
+        logger().info(f"Building ABI dumps to {self.build_dir}")
+        soong.build(["dump-ndk-abi"], env={"TARGET_PRODUCT": "ndk"})
+
+    def copy_updated_abi_dumps(self) -> None:
+        """Copies the NDK ABI dumps from the build directory to prebuilts."""
+        prebuilts_project = self.src_dir / "prebuilts/abi-dumps"
+        prebuilts_dir = prebuilts_project / "ndk"
+        abi_out = self.build_dir / "soong/abi-dumps/ndk"
+        for dump in abi_out.glob("**/abi.xml"):
+            install_path = prebuilts_dir / dump.relative_to(abi_out)
+            install_dir = install_path.parent
+            if not install_dir.exists():
+                install_dir.mkdir(parents=True)
+            logger().info(f"Copying ABI dump {dump} to {install_path}")
+            shutil.copy2(dump, install_path)
+
+    def run(self) -> None:
+        """Runs the updater.
+
+        Cleans the out directory, builds the ABI dumps, and copies the results
+        to the prebuilts directory.
+        """
+        self.build_abi_dumps()
+        self.copy_updated_abi_dumps()
+
+
+HELP = """\
+Builds and updates the NDK ABI prebuilts.
+
+Whenever a change is made that alters the NDK ABI (or an API level is
+finalized, or a new preview codename is introduced to the build), the prebuilts
+in prebuilts/abi-dumps/ndk need to be updated to match. For any finalized APIs,
+the breaking change typically needs to be reverted.
+
+Note that typically this tool should be executed via
+development/tools/ndk/update_ndk_abi.sh. That script will ensure that this tool
+is up-to-date and run with the correct arguments.
+"""
+
+
+class App:
+    """Command line application from updating NDK ABI prebuilts."""
+
+    @staticmethod
+    def parse_args() -> argparse.Namespace:
+        """Parses and returns command line arguments."""
+
+        parser = argparse.ArgumentParser(
+            formatter_class=argparse.RawDescriptionHelpFormatter, description=HELP
+        )
+
+        def resolved_path(path: str) -> Path:
+            """Converts a string into a fully resolved Path."""
+            return Path(path).resolve()
+
+        parser.add_argument(
+            "--src-dir",
+            type=resolved_path,
+            required=True,
+            help="Path to the top of the Android source tree.",
+        )
+
+        parser.add_argument(
+            "out_dir",
+            type=resolved_path,
+            metavar="OUT_DIR",
+            help="Output directory to use for building ABI dumps.",
+        )
+
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=0,
+            help="Increase logging verbosity.",
+        )
+
+        return parser.parse_args()
+
+    def run(self) -> None:
+        """Builds the new NDK ABI dumps and copies them to prebuilts."""
+        args = self.parse_args()
+        log_level = logging.DEBUG if args.verbose else logging.INFO
+        logging.basicConfig(level=log_level)
+        test_path = args.src_dir / "build/soong/soong_ui.bash"
+        if not test_path.exists():
+            sys.exit(
+                f"Source directory {args.src_dir} does not appear to be an "
+                f"Android source tree: {test_path} does not exist."
+            )
+
+        Updater(args.src_dir, args.out_dir).run()

tools/ndk/ndkabidump/soong.py

@@ -0,0 +1,112 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""APIs for interacting with Soong."""
+import logging
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+
+
+def logger() -> logging.Logger:
+    """Returns the module level logger."""
+    return logging.getLogger(__name__)
+
+
+class Soong:
+    """Interface for interacting with Soong."""
+
+    def __init__(self, build_top: Path, out_dir: Path) -> None:
+        self.out_dir = out_dir
+        self.soong_ui_path = build_top / "build/soong/soong_ui.bash"
+
+    def soong_ui(
+        self,
+        args: list[str],
+        env: dict[str, str] | None = None,
+        capture_output: bool = False,
+    ) -> str:
+        """Executes soong_ui.bash and returns the output on success.
+
+        Args:
+            args: List of string arguments to pass to soong_ui.bash.
+            env: Additional environment variables to set when running soong_ui.
+            capture_output: True if the output of the command should be captured and
+                returned. If not, the output will be printed to stdout/stderr and an
+                empty string will be returned.
+
+        Raises:
+            subprocess.CalledProcessError: The subprocess failure if the soong command
+            failed.
+
+        Returns:
+            The interleaved contents of stdout/stderr if capture_output is True, else an
+            empty string.
+        """
+        if env is None:
+            env = {}
+
+        # Use a (mostly) clean environment to avoid the caller's lunch
+        # environment affecting the build.
+        exec_env = {
+            # Newer versions of golang require the go cache, which defaults to somewhere
+            # in HOME if not set.
+            "HOME": os.environ["HOME"],
+            "OUT_DIR": str(self.out_dir.resolve()),
+            "PATH": os.environ["PATH"],
+        }
+        exec_env.update(env)
+        env_prefix = " ".join(f"{k}={v}" for k, v in exec_env.items())
+        cmd = [str(self.soong_ui_path)] + args
+        logger().debug(f"running in {os.getcwd()}: {env_prefix} {shlex.join(cmd)}")
+        result = subprocess.run(
+            cmd,
+            check=True,
+            capture_output=capture_output,
+            encoding="utf-8",
+            env=exec_env,
+        )
+        return result.stdout
+
+    def get_make_var(self, name: str) -> str:
+        """Queries the build system for the value of a make variable.
+
+        Args:
+            name: The name of the build variable to query.
+
+        Returns:
+            The value of the build variable in string form.
+        """
+        return self.soong_ui(["--dumpvar-mode", name], capture_output=True).rstrip("\n")
+
+    def clean(self) -> None:
+        """Removes the output directory, if it exists."""
+        if self.out_dir.exists():
+            shutil.rmtree(self.out_dir)
+
+    def build(self, targets: list[str], env: dict[str, str] | None = None) -> None:
+        """Builds the given targets.
+
+        The out directory will be created if it does not already exist. Existing
+        contents will not be removed, but affected outputs will be modified.
+
+        Args:
+            targets: A list of target names to build.
+            env: Additional environment variables to set when running the build.
+        """
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self.soong_ui(["--make-mode", "--soong-only"] + targets, env=env)

tools/ndk/ndkabidump/test_soong.py

@@ -0,0 +1,93 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tests for Soong APIs."""
+import contextlib
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import unittest.mock
+
+from .soong import Soong
+
+
+if "ANDROID_BUILD_TOP" not in os.environ:
+    raise RuntimeError(
+        "Cannot run Soong tests without ANDROID_BUILD_TOP defined. Run lunch."
+    )
+
+
+ANDROID_BUILD_TOP = Path(os.environ["ANDROID_BUILD_TOP"]).resolve()
+
+
+class SoongTest(unittest.TestCase):
+    """Tests for the Soong executor."""
+
+    out_dir: Path
+
+    def setUp(self) -> None:
+        with contextlib.ExitStack() as stack:
+            self.out_dir = Path(stack.enter_context(tempfile.TemporaryDirectory()))
+            self.addCleanup(stack.pop_all().close)
+
+    def test_finds_soong_ui(self) -> None:
+        """Tests that soong_ui.bash is found correctly."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        self.assertTrue(soong.soong_ui_path.exists())
+        self.assertEqual("soong_ui.bash", soong.soong_ui_path.name)
+
+    def test_get_build_var(self) -> None:
+        """Tests that we can read build variables from Soong."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        old_product = os.environ["TARGET_PRODUCT"]
+        try:
+            # Clear the lunched target out of the test environment for a
+            # consistent result.
+            del os.environ["TARGET_PRODUCT"]
+            self.assertEqual("generic", soong.get_make_var("TARGET_DEVICE"))
+        finally:
+            os.environ["TARGET_PRODUCT"] = old_product
+
+    def test_clean(self) -> None:
+        """Tests that clean works."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        self.assertTrue(self.out_dir.exists())
+        soong.clean()
+        self.assertFalse(self.out_dir.exists())
+        soong.clean()
+        self.assertFalse(self.out_dir.exists())
+
+    def test_build(self) -> None:
+        """Tests that build invokes the correct command.
+
+        Does not actually test a build, as there aren't any good options for short
+        builds in the tree, so instead tests that soong_ui is called the way we expect
+        it to be.
+        """
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        with unittest.mock.patch.object(soong, "soong_ui") as soong_ui:
+            soong.build(["foo"])
+            soong_ui.assert_called_with(
+                ["--make-mode", "--soong-only", "foo"], env=None
+            )
+
+    def test_build_creates_out_dir(self) -> None:
+        """Tests that build creates the out directory if necessary."""
+        soong = Soong(ANDROID_BUILD_TOP, self.out_dir)
+        soong.clean()
+        with unittest.mock.patch.object(soong, "soong_ui"):
+            soong.build([])
+        self.assertTrue(self.out_dir.exists())

tools/ndk/pylintrc

@@ -0,0 +1,2 @@
+[MESSAGES CONTROL]
+disable=logging-fstring-interpolation

tools/ndk/update_ndk_abi.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Entry point for NDK ABI prebuilt updater."""
+from ndkabidump import App
+
+
+if __name__ == "__main__":
+    App().run()

tools/ndk/update_ndk_abi.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -z "$ANDROID_BUILD_TOP" ]]; then
+  >&2 echo "ANDROID_BUILD_TOP not set in environment. Run lunch."
+  exit 1
+fi
+
+set -e
+set -x
+
+$ANDROID_BUILD_TOP/build/soong/soong_ui.bash --make-mode update-ndk-abi
+update-ndk-abi --src-dir $ANDROID_BUILD_TOP $ANDROID_BUILD_TOP/ndk-abi-out "$@"