Cargo2Android: add rdep tests to TEST_MAPPING

This is important because:
1. Some crates do not have their own tests enabled in presubmit.
2. As much as possible, we try to stick with one version of each
   crate. This often results in using different versions of
   dependencies than a crate has specified in its Cargo.toml.

Ensuring that a crate's tests continue to pass when its dependencies
are upgraded improves our confidence that the update is safe.

The underlying implementation uses the new Bazel queryview
to query modules and reverse dependencies.

Bug: 168167373
Test: Run cargo2android.py on a number of crates including rusqlite,
scopeguard, lock_api.

Change-Id: Id24f2d3267cf8d5e0369ece2442f8971d4ab1343

scripts/OWNERS

@@ -3,5 +3,5 @@ danalbert@google.com
 enh@google.com
 jmgao@google.com
 rprichard@google.com
-per-file add3prf.py,cargo2android.py,get_rust_pkg.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
+per-file add3prf.py,cargo2android.py,get_rust_pkg.py,update_crate_tests.py = ivanlozano@google.com,jeffv@google.com,jgalenson@google.com,mmaurer@google.com,srhines@google.com,tweek@google.com
 per-file codegen = eugenesusla@google.com

scripts/cargo2android.py

@@ -39,14 +39,15 @@ The Cargo.toml file should work at least for the host platform.
       --cargo "build --target x86_64-unknown-linux-gnu"
       --cargo "build --tests --target x86_64-unknown-linux-gnu"
 
-    Note that when there are test modules generated into Android.bp,
+    Note that when there are tests for this module or for its reverse
-    corresponding test entries will also be added into the TEST_MAPPING file.
+    dependencies, these tests will be added to the TEST_MAPPING file.
 
 If there are rustc warning messages, this script will add
 a warning comment to the owner crate module in Android.bp.
 """
 
 from __future__ import print_function
+from update_crate_tests import TestMapping
 
 import argparse
 import glob
@@ -180,36 +181,6 @@ def escape_quotes(s):  # replace '"' with '\\"'
   return s.replace('"', '\\"')
 
 
-class TestMapping(object):
-  """Entries for a TEST_MAPPING file."""
-  # Note that this only includes device tests.
-
-  def __init__(self):
-    self.entries = []
-
-  def add_test(self, name):
-    self.entries.append(name)
-
-  def is_empty(self):
-    return not self.entries
-
-  def dump(self, outf_name):
-    """Append all entries into the output file."""
-    if self.is_empty():
-      return
-    with open(outf_name, 'w') as outf:
-      outf.write('// Generated by cargo2android.py for tests in Android.bp\n')
-      outf.write('{\n  "presubmit": [\n')
-      is_first = True
-      for name in self.entries:
-        if not is_first:  # add comma and '\n' after the previous entry
-          outf.write(',\n')
-        is_first = False
-        outf.write('    {\n')
-        outf.write('      "name": "' + name + '"' + '\n    }')
-      outf.write('\n  ]\n}\n')
-
-
 class Crate(object):
   """Information of a Rust crate to collect/emit for an Android.bp module."""
 
@@ -650,14 +621,12 @@ class Crate(object):
         self.module_name = self.test_module_name()
         self.decide_one_module_type(crate_type)
         self.dump_one_android_module(crate_type)
-        # We do not add host tests, as these are handled in the Android.bp file.
       if saved_device_supported:
         self.device_supported = True
         self.host_supported = False
         self.module_name = self.test_module_name()
         self.decide_one_module_type(crate_type)
         self.dump_one_android_module(crate_type)
-        self.runner.add_test(self.outf_name, self.module_name)
       self.host_supported = saved_host_supported
       self.device_supported = saved_device_supported
       self.main_src = saved_main_src
@@ -1032,7 +1001,6 @@ class Runner(object):
 
   def __init__(self, args):
     self.bp_files = set()  # Remember all output Android.bp files.
-    self.test_mappings = {}  # Map from Android.bp file path to TestMapping.
     self.root_pkg = ''  # name of package in ./Cargo.toml
     # Saved flags, modes, and data.
     self.args = args
@@ -1184,18 +1152,11 @@ class Runner(object):
     if self.dry_run:
       print('Dry-run skip dump of TEST_MAPPING')
     else:
-      for bp_file_name in self.test_mappings:
+      test_mapping = TestMapping()
-        if bp_file_name != '/dev/null':
+      for bp_file_name in self.bp_files:
-          name = os.path.join(os.path.dirname(bp_file_name), 'TEST_MAPPING')
+        test_mapping.create_test_mapping(os.path.dirname(bp_file_name))
-          self.test_mappings[bp_file_name].dump(name)
     return self
 
-  def add_test(self, bp_file_name, test_name):
-    if bp_file_name not in self.test_mappings:
-      self.test_mappings[bp_file_name] = TestMapping()
-    mapping = self.test_mappings[bp_file_name]
-    mapping.add_test(test_name)
-
   def try_claim_module_name(self, name, owner):
     """Reserve and return True if it has not been reserved yet."""
     if name not in self.name_owners or owner == self.name_owners[name]:

scripts/update_crate_tests.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Add tests to TEST_MAPPING. Include tests for reverse dependencies."""
+import json
+import os
+import platform
+import subprocess
+import sys
+
+class Env(object):
+    def __init__(self):
+        try:
+            self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP']
+        except:
+            sys.exit('ERROR: this script must be run from an Android tree.')
+        self.cwd = os.getcwd()
+        self.cwd_relative = self.cwd.split(self.ANDROID_BUILD_TOP)[1]
+
+class Bazel(object):
+    # set up the Bazel queryview
+    def __init__(self, env):
+        os.chdir(env.ANDROID_BUILD_TOP)
+        if not os.path.exists("out/soong/queryview"):
+            print("Building Bazel Queryview. This can take a couple of minutes...")
+            cmd = "./build/soong/soong_ui.bash --build-mode --all-modules --dir=. queryview"
+            subprocess.check_output(cmd, shell=True)
+        os.chdir(env.cwd)
+
+    def path(self):
+        # Only tested on Linux.
+        if platform.system() != 'Linux':
+            sys.exit('ERROR: this script has only been tested on Linux.')
+        return "/usr/bin/bazel"
+
+    # Return all modules for a given path.
+    def query_modules(self, path):
+        with open(os.devnull, 'wb') as DEVNULL:
+            cmd = self.path() + " query --config=queryview /" + path + ":all"
+            out = subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True).strip().split("\n")
+            modules = set()
+            for line in out:
+                # speed up by excluding unused modules.
+                if "windows_x86" in line:
+                    continue
+                modules.add(line)
+            return modules
+
+    # Return all reverse dependencies for a single module.
+    def query_rdeps(self, module):
+        with open(os.devnull, 'wb') as DEVNULL:
+            # Bazel queryview special-cases external/ so we need two
+            # separate queries to collect all the reverse dependencies.
+            cmd = (self.path() + " query --config=queryview \'rdeps(//..., " +
+                    module + ")\' --output=label_kind")
+            out = (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True)
+                    .strip().split("\n"))
+            cmd = (self.path() + " query --config=queryview --universe_scope=//external/... " +
+                    "--order_output=no \"allrdeps(" + module + ")\" --output=label_kind")
+            out += (subprocess.check_output(cmd, shell=True, stderr=DEVNULL, text=True)
+                    .strip().split("\n"))
+            if '' in out:
+                out.remove('')
+            return out
+
+    # Return all reverse dependency tests for modules in this package.
+    def query_rdep_tests(self, modules):
+        rdep_tests = set()
+        print("Querying tests that depend on this crate for TEST_MAPPING. This can take a couple of minutes...")
+        for module in modules:
+            for rdep in self.query_rdeps(module):
+                rule_type, tmp, module = rdep.split(" ")
+                if rule_type == "rust_test_" or rule_type == "rust_test":
+                    rdep_tests.add(module.split(":")[1].split("--")[0])
+        return rdep_tests
+
+
+class Crate(object):
+    def __init__(self, path, bazel):
+        self.modules = bazel.query_modules(path)
+        self.rdep_tests = bazel.query_rdep_tests(self.modules)
+
+    def get_rdep_tests(self):
+        return self.rdep_tests
+
+
+class TestMapping(object):
+    def __init__(self):
+        self.env = Env()
+        self.bazel = Bazel(self.env)
+
+    def create_test_mapping(self, path):
+        tests = self.get_tests(path)
+        if not bool(tests):
+            return
+        test_mapping = self.tests_to_mapping(tests)
+        self.write_test_mapping(test_mapping)
+
+    def get_tests(self, path):
+        # for each path collect local Rust modules.
+        if path is not None and path != "":
+            return Crate(self.env.cwd_relative + "/" + path, self.bazel).get_rdep_tests()
+        else:
+            return Crate(self.env.cwd_relative, self.bazel).get_rdep_tests()
+
+    def tests_to_mapping(self, tests):
+        test_mapping = {"presubmit": []}
+        for test in tests:
+            test_mapping["presubmit"].append({"name": test})
+        return test_mapping
+
+    def write_test_mapping(self, test_mapping):
+        with open("TEST_MAPPING", "w") as json_file:
+            json_file.write("// Generated by cargo2android.py for tests that depend on this crate.\n")
+            json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True)
+            json_file.write("\n")
+
+def main():
+    TestMapping().create_test_mapping(None)
+
+if __name__ == '__main__':
+  main()