LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cargo2Android: add rdep tests to TEST_MAPPING

Browse Source 
This is important because:
1. Some crates do not have their own tests enabled in presubmit.
2. As much as possible, we try to stick with one version of each
   crate. This often results in using different versions of
   dependencies than a crate has specified in its Cargo.toml.

Ensuring that a crate's tests continue to pass when its dependencies
are upgraded improves our confidence that the update is safe.

The underlying implementation uses the new Bazel queryview
to query modules and reverse dependencies.

Bug: 168167373
Test: Run cargo2android.py on a number of crates including rusqlite,
scopeguard, lock_api.

Change-Id: Id24f2d3267cf8d5e0369ece2442f8971d4ab1343
This commit is contained in:
Jeff Vander Stoep
2021-01-15 14:15:37 +01:00
parent 169b7827ed
commit cec8bac2cb
3 changed files with 141 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/OWNERS
View File

@@ -3,5 +3,5 @@ danalbert@google.com
enh@google.com
jmgao@google.com
rprichard@google.com
per-file add3prf.py,cargo2android.py,get_rust_pkg.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
per-file add3prf.py,cargo2android.py,get_rust_pkg.py,update_crate_tests.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
per-file codegen = eugenesusla@google.com

51
scripts/cargo2android.py
View File

@@ -39,14 +39,15 @@ The Cargo.toml file should work at least for the host platform.
--cargo "build --target x86_64-unknown-linux-gnu"
--cargo "build --tests --target x86_64-unknown-linux-gnu"
Note that when there are test modules generated into Android.bp,
corresponding test entries will also be added into the TEST_MAPPING file.
Note that when there are tests for this module or for its reverse
dependencies, these tests will be added to the TEST_MAPPING file.
If there are rustc warning messages, this script will add
a warning comment to the owner crate module in Android.bp.
"""
from __future__ import print_function
from update_crate_tests import TestMapping
import argparse
import glob
@@ -180,36 +181,6 @@ def escape_quotes(s): # replace '"' with '\\"'
return s.replace('"', '\\"')
class TestMapping(object):
"""Entries for a TEST_MAPPING file."""
# Note that this only includes device tests.
def __init__(self):
self.entries = []
def add_test(self, name):
self.entries.append(name)
def is_empty(self):
return not self.entries
def dump(self, outf_name):
"""Append all entries into the output file."""
if self.is_empty():
return
with open(outf_name, 'w') as outf:
outf.write('// Generated by cargo2android.py for tests in Android.bp\n')
outf.write('{\n "presubmit": [\n')
is_first = True
for name in self.entries:
if not is_first: # add comma and '\n' after the previous entry
outf.write(',\n')
is_first = False
outf.write(' {\n')
outf.write(' "name": "' + name + '"' + '\n }')
outf.write('\n ]\n}\n')
class Crate(object):
"""Information of a Rust crate to collect/emit for an Android.bp module."""
@@ -650,14 +621,12 @@ class Crate(object):
self.module_name = self.test_module_name()
self.decide_one_module_type(crate_type)
self.dump_one_android_module(crate_type)
# We do not add host tests, as these are handled in the Android.bp file.
if saved_device_supported:
self.device_supported = True
self.host_supported = False
self.module_name = self.test_module_name()
self.decide_one_module_type(crate_type)
self.dump_one_android_module(crate_type)
self.runner.add_test(self.outf_name, self.module_name)
self.host_supported = saved_host_supported
self.device_supported = saved_device_supported
self.main_src = saved_main_src
@@ -1032,7 +1001,6 @@ class Runner(object):
def __init__(self, args):
self.bp_files = set() # Remember all output Android.bp files.
self.test_mappings = {} # Map from Android.bp file path to TestMapping.
self.root_pkg = '' # name of package in ./Cargo.toml
# Saved flags, modes, and data.
self.args = args
@@ -1184,18 +1152,11 @@ class Runner(object):
if self.dry_run:
print('Dry-run skip dump of TEST_MAPPING')
else:
for bp_file_name in self.test_mappings:
if bp_file_name != '/dev/null':
name = os.path.join(os.path.dirname(bp_file_name), 'TEST_MAPPING')
self.test_mappings[bp_file_name].dump(name)
test_mapping = TestMapping()
for bp_file_name in self.bp_files:
test_mapping.create_test_mapping(os.path.dirname(bp_file_name))
return self
def add_test(self, bp_file_name, test_name):
if bp_file_name not in self.test_mappings:
self.test_mappings[bp_file_name] = TestMapping()
mapping = self.test_mappings[bp_file_name]
mapping.add_test(test_name)
def try_claim_module_name(self, name, owner):
"""Reserve and return True if it has not been reserved yet."""
if name not in self.name_owners or owner == self.name_owners[name]:

134
scripts/update_crate_tests.py Executable file
View File

@@ -0,0 +1,134 @@
#!/usr/bin/env python3
#
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Add tests to TEST_MAPPING. Include tests for reverse dependencies."""
import json
import os
import platform
import subprocess
import sys
class Env(object):
def __init__(self):
try:
self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP']
except:
sys.exit('ERROR: this script must be run from an Android tree.')
self.cwd = os.getcwd()
self.cwd_relative = self.cwd.split(self.ANDROID_BUILD_TOP)[1]
class Bazel(object):
# set up the Bazel queryview
def __init__(self, env):
os.chdir(env.ANDROID_BUILD_TOP)
if not os.path.exists("out/soong/queryview"):
print("Building Bazel Queryview. This can take a couple of minutes...")
cmd = "./build/soong/soong_ui.bash --build-mode --all-modules --dir=. queryview"
subprocess.check_output(cmd, shell=True)
os.chdir(env.cwd)
def path(self):
# Only tested on Linux.
if platform.system() != 'Linux':
sys.exit('ERROR: this script has only been tested on Linux.')
return "/usr/bin/bazel"
# Return all modules for a given path.
def query_modules(self, path):
with open(os.devnull, 'wb') as DEVNULL:
cmd = self.path() + " query --config=queryview /" + path + ":all"
out = subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True).strip().split("\n")
modules = set()
for line in out:
# speed up by excluding unused modules.
if "windows_x86" in line:
continue
modules.add(line)
return modules
# Return all reverse dependencies for a single module.
def query_rdeps(self, module):
with open(os.devnull, 'wb') as DEVNULL:
# Bazel queryview special-cases external/ so we need two
# separate queries to collect all the reverse dependencies.
cmd = (self.path() + " query --config=queryview \'rdeps(//..., " +
module + ")\' --output=label_kind")
out = (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True)
.strip().split("\n"))
cmd = (self.path() + " query --config=queryview --universe_scope=//external/... " +
"--order_output=no \"allrdeps(" + module + ")\" --output=label_kind")
out += (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True)
.strip().split("\n"))
if '' in out:
out.remove('')
return out
# Return all reverse dependency tests for modules in this package.
def query_rdep_tests(self, modules):
rdep_tests = set()
print("Querying tests that depend on this crate for TEST_MAPPING. This can take a couple of minutes...")
for module in modules:
for rdep in self.query_rdeps(module):
rule_type, tmp, module = rdep.split(" ")
if rule_type == "rust_test_" or rule_type == "rust_test":
rdep_tests.add(module.split(":")[1].split("--")[0])
return rdep_tests
class Crate(object):
def __init__(self, path, bazel):
self.modules = bazel.query_modules(path)
self.rdep_tests = bazel.query_rdep_tests(self.modules)
def get_rdep_tests(self):
return self.rdep_tests
class TestMapping(object):
def __init__(self):
self.env = Env()
self.bazel = Bazel(self.env)
def create_test_mapping(self, path):
tests = self.get_tests(path)
if not bool(tests):
return
test_mapping = self.tests_to_mapping(tests)
self.write_test_mapping(test_mapping)
def get_tests(self, path):
# for each path collect local Rust modules.
if path is not None and path != "":
return Crate(self.env.cwd_relative + "/" + path, self.bazel).get_rdep_tests()
else:
return Crate(self.env.cwd_relative, self.bazel).get_rdep_tests()
def tests_to_mapping(self, tests):
test_mapping = {"presubmit": []}
for test in tests:
test_mapping["presubmit"].append({"name": test})
return test_mapping
def write_test_mapping(self, test_mapping):
with open("TEST_MAPPING", "w") as json_file:
json_file.write("// Generated by cargo2android.py for tests that depend on this crate.\n")
json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True)
json_file.write("\n")
def main():
TestMapping().create_test_mapping(None)
if __name__ == '__main__':
main()