diff --git a/scripts/OWNERS b/scripts/OWNERS index 87a9c4d2a..3e9b36a49 100644 --- a/scripts/OWNERS +++ b/scripts/OWNERS @@ -3,5 +3,5 @@ danalbert@google.com enh@google.com jmgao@google.com rprichard@google.com -per-file add3prf.py,cargo2android.py,get_rust_pkg.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com +per-file add3prf.py,cargo2android.py,get_rust_pkg.py,update_crate_tests.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com per-file codegen = eugenesusla@google.com diff --git a/scripts/cargo2android.py b/scripts/cargo2android.py index d1c666a1a..f32061e72 100755 --- a/scripts/cargo2android.py +++ b/scripts/cargo2android.py @@ -39,14 +39,15 @@ The Cargo.toml file should work at least for the host platform. --cargo "build --target x86_64-unknown-linux-gnu" --cargo "build --tests --target x86_64-unknown-linux-gnu" - Note that when there are test modules generated into Android.bp, - corresponding test entries will also be added into the TEST_MAPPING file. + Note that when there are tests for this module or for its reverse + dependencies, these tests will be added to the TEST_MAPPING file. If there are rustc warning messages, this script will add a warning comment to the owner crate module in Android.bp. """ from __future__ import print_function +from update_crate_tests import TestMapping import argparse import glob @@ -180,36 +181,6 @@ def escape_quotes(s): # replace '"' with '\\"' return s.replace('"', '\\"') -class TestMapping(object): - """Entries for a TEST_MAPPING file.""" - # Note that this only includes device tests. - - def __init__(self): - self.entries = [] - - def add_test(self, name): - self.entries.append(name) - - def is_empty(self): - return not self.entries - - def dump(self, outf_name): - """Append all entries into the output file.""" - if self.is_empty(): - return - with open(outf_name, 'w') as outf: - outf.write('// Generated by cargo2android.py for tests in Android.bp\n') - outf.write('{\n "presubmit": [\n') - is_first = True - for name in self.entries: - if not is_first: # add comma and '\n' after the previous entry - outf.write(',\n') - is_first = False - outf.write(' {\n') - outf.write(' "name": "' + name + '"' + '\n }') - outf.write('\n ]\n}\n') - - class Crate(object): """Information of a Rust crate to collect/emit for an Android.bp module.""" @@ -650,14 +621,12 @@ class Crate(object): self.module_name = self.test_module_name() self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) - # We do not add host tests, as these are handled in the Android.bp file. if saved_device_supported: self.device_supported = True self.host_supported = False self.module_name = self.test_module_name() self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) - self.runner.add_test(self.outf_name, self.module_name) self.host_supported = saved_host_supported self.device_supported = saved_device_supported self.main_src = saved_main_src @@ -1032,7 +1001,6 @@ class Runner(object): def __init__(self, args): self.bp_files = set() # Remember all output Android.bp files. - self.test_mappings = {} # Map from Android.bp file path to TestMapping. self.root_pkg = '' # name of package in ./Cargo.toml # Saved flags, modes, and data. self.args = args @@ -1184,18 +1152,11 @@ class Runner(object): if self.dry_run: print('Dry-run skip dump of TEST_MAPPING') else: - for bp_file_name in self.test_mappings: - if bp_file_name != '/dev/null': - name = os.path.join(os.path.dirname(bp_file_name), 'TEST_MAPPING') - self.test_mappings[bp_file_name].dump(name) + test_mapping = TestMapping() + for bp_file_name in self.bp_files: + test_mapping.create_test_mapping(os.path.dirname(bp_file_name)) return self - def add_test(self, bp_file_name, test_name): - if bp_file_name not in self.test_mappings: - self.test_mappings[bp_file_name] = TestMapping() - mapping = self.test_mappings[bp_file_name] - mapping.add_test(test_name) - def try_claim_module_name(self, name, owner): """Reserve and return True if it has not been reserved yet.""" if name not in self.name_owners or owner == self.name_owners[name]: diff --git a/scripts/update_crate_tests.py b/scripts/update_crate_tests.py new file mode 100755 index 000000000..0e2437193 --- /dev/null +++ b/scripts/update_crate_tests.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Add tests to TEST_MAPPING. Include tests for reverse dependencies.""" +import json +import os +import platform +import subprocess +import sys + +class Env(object): + def __init__(self): + try: + self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP'] + except: + sys.exit('ERROR: this script must be run from an Android tree.') + self.cwd = os.getcwd() + self.cwd_relative = self.cwd.split(self.ANDROID_BUILD_TOP)[1] + +class Bazel(object): + # set up the Bazel queryview + def __init__(self, env): + os.chdir(env.ANDROID_BUILD_TOP) + if not os.path.exists("out/soong/queryview"): + print("Building Bazel Queryview. This can take a couple of minutes...") + cmd = "./build/soong/soong_ui.bash --build-mode --all-modules --dir=. queryview" + subprocess.check_output(cmd, shell=True) + os.chdir(env.cwd) + + def path(self): + # Only tested on Linux. + if platform.system() != 'Linux': + sys.exit('ERROR: this script has only been tested on Linux.') + return "/usr/bin/bazel" + + # Return all modules for a given path. + def query_modules(self, path): + with open(os.devnull, 'wb') as DEVNULL: + cmd = self.path() + " query --config=queryview /" + path + ":all" + out = subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True).strip().split("\n") + modules = set() + for line in out: + # speed up by excluding unused modules. + if "windows_x86" in line: + continue + modules.add(line) + return modules + + # Return all reverse dependencies for a single module. + def query_rdeps(self, module): + with open(os.devnull, 'wb') as DEVNULL: + # Bazel queryview special-cases external/ so we need two + # separate queries to collect all the reverse dependencies. + cmd = (self.path() + " query --config=queryview \'rdeps(//..., " + + module + ")\' --output=label_kind") + out = (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True) + .strip().split("\n")) + cmd = (self.path() + " query --config=queryview --universe_scope=//external/... " + + "--order_output=no \"allrdeps(" + module + ")\" --output=label_kind") + out += (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True) + .strip().split("\n")) + if '' in out: + out.remove('') + return out + + # Return all reverse dependency tests for modules in this package. + def query_rdep_tests(self, modules): + rdep_tests = set() + print("Querying tests that depend on this crate for TEST_MAPPING. This can take a couple of minutes...") + for module in modules: + for rdep in self.query_rdeps(module): + rule_type, tmp, module = rdep.split(" ") + if rule_type == "rust_test_" or rule_type == "rust_test": + rdep_tests.add(module.split(":")[1].split("--")[0]) + return rdep_tests + + +class Crate(object): + def __init__(self, path, bazel): + self.modules = bazel.query_modules(path) + self.rdep_tests = bazel.query_rdep_tests(self.modules) + + def get_rdep_tests(self): + return self.rdep_tests + + +class TestMapping(object): + def __init__(self): + self.env = Env() + self.bazel = Bazel(self.env) + + def create_test_mapping(self, path): + tests = self.get_tests(path) + if not bool(tests): + return + test_mapping = self.tests_to_mapping(tests) + self.write_test_mapping(test_mapping) + + def get_tests(self, path): + # for each path collect local Rust modules. + if path is not None and path != "": + return Crate(self.env.cwd_relative + "/" + path, self.bazel).get_rdep_tests() + else: + return Crate(self.env.cwd_relative, self.bazel).get_rdep_tests() + + def tests_to_mapping(self, tests): + test_mapping = {"presubmit": []} + for test in tests: + test_mapping["presubmit"].append({"name": test}) + return test_mapping + + def write_test_mapping(self, test_mapping): + with open("TEST_MAPPING", "w") as json_file: + json_file.write("// Generated by cargo2android.py for tests that depend on this crate.\n") + json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True) + json_file.write("\n") + +def main(): + TestMapping().create_test_mapping(None) + +if __name__ == '__main__': + main()