From fc485b2384f03d8226266dbf3bff856f5f96b10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=C3=A9baud=20Weksteen?= Date: Thu, 10 Jun 2021 13:30:20 +0200 Subject: [PATCH 1/2] update_crate_tests: Remove cwd Env.cwd is misleading as it captures the location of the crate/package being updated. Move the detection logic to the Package class and rename the attribute. There is the expectation that before running soong_ui, cwd is set to ANDROID_BUILD_TOP; and before running write_mapping, that cwd is set to the package directory. Add comments to capture this. Test: update_crate_tests.py in external/rust/crates/libc Change-Id: I5c4f16afe0b92b7c85173e4149ee17dd39c773c2 --- scripts/update_crate_tests.py | 42 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/scripts/update_crate_tests.py b/scripts/update_crate_tests.py index 5ad95ab31..3f4bb92d7 100755 --- a/scripts/update_crate_tests.py +++ b/scripts/update_crate_tests.py @@ -41,24 +41,13 @@ class UpdaterException(Exception): class Env(object): - def __init__(self, path): + def __init__(self): try: self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP'] except KeyError: raise UpdaterException('$ANDROID_BUILD_TOP is not defined; you ' 'must first source build/envsetup.sh and ' 'select a target.') - if path == None: - self.cwd = os.getcwd() - else: - self.cwd = path - try: - self.cwd_relative = self.cwd.split(self.ANDROID_BUILD_TOP)[1] - except IndexError: - raise UpdaterException('The path ' + self.cwd + ' is not under ' + - self.ANDROID_BUILD_TOP + '; You must be in the ' - 'directory of a crate or pass its absolute path ' - 'as first argument.') class Bazel(object): @@ -68,6 +57,8 @@ class Bazel(object): raise UpdaterException('This script has only been tested on Linux.') self.path = os.path.join(env.ANDROID_BUILD_TOP, "tools", "bazel") soong_ui = os.path.join(env.ANDROID_BUILD_TOP, "build", "soong", "soong_ui.bash") + + # soong_ui requires to be at the root of the repository. os.chdir(env.ANDROID_BUILD_TOP) print("Generating Bazel files...") cmd = [soong_ui, "--make-mode", "GENERATE_BAZEL_FILES=1", "nothing"] @@ -82,7 +73,6 @@ class Bazel(object): subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True) except subprocess.CalledProcessError as e: raise UpdaterException('Unable to update TEST_MAPPING: ' + e.output) - os.chdir(env.cwd) # Return all modules for a given path. def query_modules(self, path): @@ -125,8 +115,22 @@ class Bazel(object): class Package(object): - def __init__(self, path, bazel): - modules = bazel.query_modules(path) + def __init__(self, path, env, bazel): + if path == None: + self.dir = os.getcwd() + else: + self.dir = path + try: + self.dir_rel = self.dir.split(env.ANDROID_BUILD_TOP)[1] + except IndexError: + raise UpdaterException('The path ' + self.dir + ' is not under ' + + env.ANDROID_BUILD_TOP + '; You must be in the ' + 'directory of a crate or pass its absolute path ' + 'as first argument.') + + # Move to the package_directory. + os.chdir(self.dir) + modules = bazel.query_modules(self.dir_rel) self.rdep_tests = bazel.query_rdep_tests(modules) def get_rdep_tests(self): @@ -135,11 +139,12 @@ class Package(object): class TestMapping(object): def __init__(self, path): - self.env = Env(path) - self.bazel = Bazel(self.env) + env = Env() + bazel = Bazel(env) + self.package = Package(path, env, bazel) def create(self): - tests = Package(self.env.cwd_relative, self.bazel).get_rdep_tests() + tests = self.package.get_rdep_tests() if not bool(tests): return test_mapping = self.tests_to_mapping(tests) @@ -164,6 +169,7 @@ class TestMapping(object): json_file.write("\n") print("TEST_MAPPING successfully updated!") + def main(): if len(sys.argv) == 2: path = sys.argv[1] From 3604b754562a972ccc16d50f9b50060e2c60f750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=C3=A9baud=20Weksteen?= Date: Thu, 10 Jun 2021 14:22:00 +0200 Subject: [PATCH 2/2] update_crate_tests: Add documentation Test: pydoc update_crate_tests Change-Id: Ice2c3a8ab901e38b2cf77433ec1ba6aac2360236 --- scripts/update_crate_tests.py | 108 +++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 15 deletions(-) diff --git a/scripts/update_crate_tests.py b/scripts/update_crate_tests.py index 3f4bb92d7..192f50ee8 100755 --- a/scripts/update_crate_tests.py +++ b/scripts/update_crate_tests.py @@ -13,34 +13,61 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Add tests to TEST_MAPPING. Include tests for reverse dependencies.""" +"""Add or update tests to TEST_MAPPING. + +This script uses Bazel to find reverse dependencies on a crate and generates a +TEST_MAPPING file. It accepts the absolute path to a crate as argument. If no +argument is provided, it assumes the crate is the current directory. + + Usage: + $ . build/envsetup.sh + $ lunch aosp_arm64-eng + $ update_crate_tests.py $ANDROID_BUILD_TOP/external/rust/crates/libc + +This script is automatically called by external_updater. +""" + import json import os import platform import subprocess import sys -test_options = { +# Some tests requires specific options. Consider fixing the upstream crate +# before updating this dictionary. +TEST_OPTIONS = { "ring_device_test_tests_digest_tests": [{"test-timeout": "600000"}], "ring_device_test_src_lib": [{"test-timeout": "100000"}], } -test_exclude = [ + +# Excluded tests. These tests will be ignored by this script. +TEST_EXCLUDE = [ "aidl_test_rust_client", "aidl_test_rust_service" - ] -exclude_paths = [ +] + +# Excluded modules. +EXCLUDE_PATHS = [ "//external/adhd", "//external/crosvm", "//external/libchromeos-rs", "//external/vm_tools" - ] +] class UpdaterException(Exception): - pass + """Exception generated by this script.""" class Env(object): + """Env captures the execution environment. + + It ensures this script is executed within an AOSP repository. + + Attributes: + ANDROID_BUILD_TOP: A string representing the absolute path to the top + of the repository. + """ def __init__(self): try: self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP'] @@ -51,8 +78,25 @@ class Env(object): class Bazel(object): - # set up the Bazel queryview + """Bazel wrapper. + + The wrapper is used to call bazel queryview and generate the list of + reverse dependencies. + + Attributes: + path: The path to the bazel executable. + """ def __init__(self, env): + """Constructor. + + Note that the current directory is changed to ANDROID_BUILD_TOP. + + Args: + env: An instance of Env. + + Raises: + UpdaterException: an error occurred while calling soong_ui. + """ if platform.system() != 'Linux': raise UpdaterException('This script has only been tested on Linux.') self.path = os.path.join(env.ANDROID_BUILD_TOP, "tools", "bazel") @@ -74,8 +118,8 @@ class Bazel(object): except subprocess.CalledProcessError as e: raise UpdaterException('Unable to update TEST_MAPPING: ' + e.output) - # Return all modules for a given path. def query_modules(self, path): + """Returns all modules for a given path.""" cmd = self.path + " query --config=queryview /" + path + ":all" out = subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True).strip().split("\n") modules = set() @@ -86,8 +130,8 @@ class Bazel(object): modules.add(line) return modules - # Return all reverse dependencies for a single module. def query_rdeps(self, module): + """Returns all reverse dependencies for a single module.""" cmd = (self.path + " query --config=queryview \'rdeps(//..., " + module + ")\' --output=label_kind") out = (subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True) @@ -97,13 +141,13 @@ class Bazel(object): return out def exclude_module(self, module): - for path in exclude_paths: + for path in EXCLUDE_PATHS: if module.startswith(path): return True return False - # Return all reverse dependency tests for modules in this package. def query_rdep_tests(self, modules): + """Returns all reverse dependency tests for modules in this package.""" rdep_tests = set() for module in modules: for rdep in self.query_rdeps(module): @@ -115,7 +159,28 @@ class Bazel(object): class Package(object): + """A Bazel package. + + Attributes: + dir: The absolute path to this package. + dir_rel: The relative path to this package. + rdep_tests: The list of computed reverse dependencies. + """ def __init__(self, path, env, bazel): + """Constructor. + + Note that the current directory is changed to the package location when + called. + + Args: + path: Path to the package. + env: An instance of Env. + bazel: An instance of Bazel. + + Raises: + UpdaterException: the package does not appear to belong to the + current repository. + """ if path == None: self.dir = os.getcwd() else: @@ -138,12 +203,23 @@ class Package(object): class TestMapping(object): + """A TEST_MAPPING file. + + Attributes: + package: The package associated with this TEST_MAPPING file. + """ def __init__(self, path): + """Constructor. + + Args: + path: The absolute path to the package. + """ env = Env() bazel = Bazel(env) self.package = Package(path, env, bazel) def create(self): + """Generates the TEST_MAPPING file.""" tests = self.package.get_rdep_tests() if not bool(tests): return @@ -151,18 +227,20 @@ class TestMapping(object): self.write_test_mapping(test_mapping) def tests_to_mapping(self, tests): + """Translate the test list into a dictionary.""" test_mapping = {"presubmit": []} for test in tests: - if test in test_exclude: + if test in TEST_EXCLUDE: continue - if test in test_options: - test_mapping["presubmit"].append({"name": test, "options": test_options[test]}) + if test in TEST_OPTIONS: + test_mapping["presubmit"].append({"name": test, "options": TEST_OPTIONS[test]}) else: test_mapping["presubmit"].append({"name": test}) test_mapping["presubmit"] = sorted(test_mapping["presubmit"], key=lambda t: t["name"]) return test_mapping def write_test_mapping(self, test_mapping): + """Writes the TEST_MAPPING file.""" with open("TEST_MAPPING", "w") as json_file: json_file.write("// Generated by update_crate_tests.py for tests that depend on this crate.\n") json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True)