Add jarjar rules generator

(This rolls forward part of a previous change, now that jarjar was fixed
to not get very slow when the number of rules increases).

Jarjar rules are hard to keep in sync with code, and hard to maintain
manually as the distinction between what should and should not be
jarjared is not always clear. This results in unsafe binaries that are
manually maintained, and developer frustration when something fails due
to incorrect jarjar rules.

Add utility to autogenerate jarjar rules, which can be run at build time
time (via a genrule) instead. The generator scans pre-jarjar
intermediate artifacts, and outputs jarjar rules for every class to put
it in a package specific to the module. The only exceptions are:

 - Classes that are API (module-lib API is the largest API surface of
   the module, so module-lib API stubs would typically be used)
 - Classes that have unsupportedappusage symbols
 - Classes that are excluded manually (for example, because they have
   hardcoded external references, like for
   ConnectivityServiceInitializer in SystemServer).

Bug: 217129444
Test: atest jarjar-rules-generator-test;

Change-Id: I3493957e39a661b6c2e330944e7c3023b8f3203e

framework/jarjar-excludes.txt

@@ -0,0 +1,25 @@
+# INetworkStatsProvider / INetworkStatsProviderCallback are referenced from net-tests-utils, which
+# may be used by tests that do not apply connectivity jarjar rules.
+# TODO: move files to a known internal package (like android.net.connectivity.visiblefortesting)
+# so that they do not need jarjar
+android\.net\.netstats\.provider\.INetworkStatsProvider(\$.+)?
+android\.net\.netstats\.provider\.INetworkStatsProviderCallback(\$.+)?
+
+# INetworkAgent / INetworkAgentRegistry are used in NetworkAgentTest
+# TODO: move files to android.net.connectivity.visiblefortesting
+android\.net\.INetworkAgent(\$.+)?
+android\.net\.INetworkAgentRegistry(\$.+)?
+
+# IConnectivityDiagnosticsCallback used in ConnectivityDiagnosticsManagerTest
+# TODO: move files to android.net.connectivity.visiblefortesting
+android\.net\.IConnectivityDiagnosticsCallback(\$.+)?
+
+
+# KeepaliveUtils is used by ConnectivityManager CTS
+# TODO: move into service-connectivity so framework-connectivity stops using
+# ServiceConnectivityResources (callers need high permissions to find/query the resource apk anyway)
+# and have a ConnectivityManager test API instead
+android\.net\.util\.KeepaliveUtils(\$.+)?
+
+# TODO (b/217115866): add jarjar rules for Nearby
+android\.nearby\..+

service/jarjar-excludes.txt

@@ -0,0 +1,9 @@
+# Classes loaded by SystemServer via their hardcoded name, so they can't be jarjared
+com\.android\.server\.ConnectivityServiceInitializer(\$.+)?
+com\.android\.server\.NetworkStatsServiceInitializer(\$.+)?
+
+# Do not jarjar com.android.server, as several unit tests fail because they lose
+# package-private visibility between jarjared and non-jarjared classes.
+# TODO: fix the tests and also jarjar com.android.server, or at least only exclude a package that
+# is specific to the module like com.android.server.connectivity
+com\.android\.server\..+

tools/Android.bp

@@ -0,0 +1,91 @@
+//
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+    // See: http://go/android-license-faq
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+// Build tool used to generate jarjar rules for all classes in a jar, except those that are
+// API, UnsupportedAppUsage or otherwise excluded.
+python_binary_host {
+    name: "jarjar-rules-generator",
+    srcs: [
+        "gen_jarjar.py",
+    ],
+    main: "gen_jarjar.py",
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+        },
+    },
+    visibility: ["//packages/modules/Connectivity:__subpackages__"],
+}
+
+genrule_defaults {
+    name: "jarjar-rules-combine-defaults",
+    // Concat files with a line break in the middle
+    cmd: "for src in $(in); do cat $${src}; echo; done > $(out)",
+    defaults_visibility: ["//packages/modules/Connectivity:__subpackages__"],
+}
+
+java_library {
+    name: "jarjar-rules-generator-testjavalib",
+    srcs: ["testdata/java/**/*.java"],
+    visibility: ["//visibility:private"],
+}
+
+// TODO(b/233723405) - Remove this workaround.
+// Temporary work around of b/233723405. Using the module_lib stub directly
+// in the test causes it to sometimes get the dex jar and sometimes get the
+// classes jar due to b/233111644. Statically including it here instead
+// ensures that it will always get the classes jar.
+java_library {
+    name: "framework-connectivity.stubs.module_lib-for-test",
+    visibility: ["//visibility:private"],
+    static_libs: [
+        "framework-connectivity.stubs.module_lib",
+    ],
+    // Not strictly necessary but specified as this MUST not have generate
+    // a dex jar as that will break the tests.
+    compile_dex: false,
+}
+
+python_test_host {
+    name: "jarjar-rules-generator-test",
+    srcs: [
+        "gen_jarjar.py",
+        "gen_jarjar_test.py",
+    ],
+    data: [
+        "testdata/test-jarjar-excludes.txt",
+        "testdata/test-unsupportedappusage.txt",
+        ":framework-connectivity.stubs.module_lib-for-test",
+        ":jarjar-rules-generator-testjavalib",
+    ],
+    main: "gen_jarjar_test.py",
+    version: {
+        py2: {
+            enabled: false,
+        },
+        py3: {
+            enabled: true,
+        },
+    },
+}

tools/gen_jarjar.py

@@ -0,0 +1,133 @@
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" This script generates jarjar rule files to add a jarjar prefix to all classes, except those
+that are API, unsupported API or otherwise excluded."""
+
+import argparse
+import io
+import re
+import subprocess
+from xml import sax
+from xml.sax.handler import ContentHandler
+from zipfile import ZipFile
+
+
+def parse_arguments(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--jars', nargs='+',
+        help='Path to pre-jarjar JAR. Can be followed by multiple space-separated paths.')
+    parser.add_argument(
+        '--prefix', required=True,
+        help='Package prefix to use for jarjared classes, '
+             'for example "com.android.connectivity" (does not end with a dot).')
+    parser.add_argument(
+        '--output', required=True, help='Path to output jarjar rules file.')
+    parser.add_argument(
+        '--apistubs', nargs='*', default=[],
+        help='Path to API stubs jar. Classes that are API will not be jarjared. Can be followed by '
+             'multiple space-separated paths.')
+    parser.add_argument(
+        '--unsupportedapi', nargs='*', default=[],
+        help='Path to UnsupportedAppUsage hidden API .txt lists. '
+             'Classes that have UnsupportedAppUsage API will not be jarjared. Can be followed by '
+             'multiple space-separated paths.')
+    parser.add_argument(
+        '--excludes', nargs='*', default=[],
+        help='Path to files listing classes that should not be jarjared. Can be followed by '
+             'multiple space-separated paths. '
+             'Each file should contain one full-match regex per line. Empty lines or lines '
+             'starting with "#" are ignored.')
+    return parser.parse_args(argv)
+
+
+def _list_toplevel_jar_classes(jar):
+    """List all classes in a .class .jar file that are not inner classes."""
+    return {_get_toplevel_class(c) for c in _list_jar_classes(jar)}
+
+def _list_jar_classes(jar):
+    with ZipFile(jar, 'r') as zip:
+        files = zip.namelist()
+        assert 'classes.dex' not in files, f'Jar file {jar} is dexed, ' \
+                                           'expected an intermediate zip of .class files'
+        class_len = len('.class')
+        return [f.replace('/', '.')[:-class_len] for f in files
+                if f.endswith('.class') and not f.endswith('/package-info.class')]
+
+
+def _list_hiddenapi_classes(txt_file):
+    out = set()
+    with open(txt_file, 'r') as f:
+        for line in f:
+            if not line.strip():
+                continue
+            assert line.startswith('L') and ';' in line, f'Class name not recognized: {line}'
+            clazz = line.replace('/', '.').split(';')[0][1:]
+            out.add(_get_toplevel_class(clazz))
+    return out
+
+
+def _get_toplevel_class(clazz):
+    """Return the name of the toplevel (not an inner class) enclosing class of the given class."""
+    if '$' not in clazz:
+        return clazz
+    return clazz.split('$')[0]
+
+
+def _get_excludes(path):
+    out = []
+    with open(path, 'r') as f:
+        for line in f:
+            stripped = line.strip()
+            if not stripped or stripped.startswith('#'):
+                continue
+            out.append(re.compile(stripped))
+    return out
+
+
+def make_jarjar_rules(args):
+    excluded_classes = set()
+    for apistubs_file in args.apistubs:
+        excluded_classes.update(_list_toplevel_jar_classes(apistubs_file))
+
+    for unsupportedapi_file in args.unsupportedapi:
+        excluded_classes.update(_list_hiddenapi_classes(unsupportedapi_file))
+
+    exclude_regexes = []
+    for exclude_file in args.excludes:
+        exclude_regexes.extend(_get_excludes(exclude_file))
+
+    with open(args.output, 'w') as outfile:
+        for jar in args.jars:
+            jar_classes = _list_jar_classes(jar)
+            jar_classes.sort()
+            for clazz in jar_classes:
+                if (_get_toplevel_class(clazz) not in excluded_classes and
+                        not any(r.fullmatch(clazz) for r in exclude_regexes)):
+                    outfile.write(f'rule {clazz} {args.prefix}.@0\n')
+                    # Also include jarjar rules for unit tests of the class, so the package matches
+                    outfile.write(f'rule {clazz}Test {args.prefix}.@0\n')
+                    outfile.write(f'rule {clazz}Test$* {args.prefix}.@0\n')
+
+
+def _main():
+    # Pass in None to use argv
+    args = parse_arguments(None)
+    make_jarjar_rules(args)
+
+
+if __name__ == '__main__':
+    _main()

tools/gen_jarjar_test.py

@@ -0,0 +1,57 @@
+#  Copyright (C) 2022 The Android Open Source Project
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import gen_jarjar
+import unittest
+
+
+class TestGenJarjar(unittest.TestCase):
+    def test_gen_rules(self):
+        args = gen_jarjar.parse_arguments([
+            "--jars", "jarjar-rules-generator-testjavalib.jar",
+            "--prefix", "jarjar.prefix",
+            "--output", "test-output-rules.txt",
+            "--apistubs", "framework-connectivity.stubs.module_lib.jar",
+            "--unsupportedapi", "testdata/test-unsupportedappusage.txt",
+            "--excludes", "testdata/test-jarjar-excludes.txt",
+        ])
+        gen_jarjar.make_jarjar_rules(args)
+
+        with open(args.output) as out:
+            lines = out.readlines()
+
+        self.assertListEqual([
+            'rule test.utils.TestUtilClass jarjar.prefix.@0\n',
+            'rule test.utils.TestUtilClassTest jarjar.prefix.@0\n',
+            'rule test.utils.TestUtilClassTest$* jarjar.prefix.@0\n',
+            'rule test.utils.TestUtilClass$TestInnerClass jarjar.prefix.@0\n',
+            'rule test.utils.TestUtilClass$TestInnerClassTest jarjar.prefix.@0\n',
+            'rule test.utils.TestUtilClass$TestInnerClassTest$* jarjar.prefix.@0\n'], lines)
+
+
+if __name__ == '__main__':
+    # Need verbosity=2 for the test results parser to find results
+    unittest.main(verbosity=2)

tools/testdata/java/android/net/LinkProperties.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net;
+
+/**
+ * Test class with a name matching a public API.
+ */
+public class LinkProperties {
+}

tools/testdata/java/test/jarjarexcluded/JarjarExcludedClass.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.jarjarexcluded;
+
+/**
+ * Test class that is excluded from jarjar.
+ */
+public class JarjarExcludedClass {
+}

tools/testdata/java/test/unsupportedappusage/TestUnsupportedAppUsageClass.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.unsupportedappusage;
+
+public class TestUnsupportedAppUsageClass {
+    public void testMethod() {}
+}

tools/testdata/java/test/utils/TestUtilClass.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.utils;
+
+/**
+ * Sample class to test jarjar rules.
+ */
+public class TestUtilClass {
+    public static class TestInnerClass {}
+}

tools/testdata/test-jarjar-excludes.txt

@@ -0,0 +1,3 @@
+# Test file for excluded classes
+test\.jarj.rexcluded\.JarjarExcludedCla.s
+test\.jarjarexcluded\.JarjarExcludedClass\$TestInnerCl.ss

tools/testdata/test-unsupportedappusage.txt

@@ -0,0 +1 @@
+Ltest/unsupportedappusage/TestUnsupportedAppUsageClass;->testMethod()V