Merge "VNDK snapshot tool generates license information from json" am: c973f68913 am: b7e7077c5e

Original change: https://android-review.googlesource.com/c/platform/development/+/2539710

Change-Id: I98d372acda2b5eab1a59e6eef58fb96f39668a6e
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

vndk/snapshot/check_gpl_license.py

@@ -36,14 +36,15 @@ class GPLChecker(object):
     MANIFEST_XML = utils.MANIFEST_FILE_NAME
     MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
 
-    def __init__(self, install_dir, android_build_top, temp_artifact_dir,
+    def __init__(self, install_dir, android_build_top, gpl_projects,
-                 remote_git):
+                 temp_artifact_dir, remote_git):
         """GPLChecker constructor.
 
         Args:
           install_dir: string, absolute path to the prebuilts/vndk/v{version}
             directory where the build files will be generated.
           android_build_top: string, absolute path to ANDROID_BUILD_TOP
+          gpl_projects: list of strings, names of libraries under GPL
           temp_artifact_dir: string, temp directory to hold build artifacts
             fetched from Android Build server.
           remote_git: string, remote name to fetch and check if the revision of
@@ -53,10 +54,9 @@ class GPLChecker(object):
         self._android_build_top = android_build_top
         self._install_dir = install_dir
         self._remote_git = remote_git
+        self._gpl_projects = gpl_projects
         self._manifest_file = os.path.join(temp_artifact_dir,
                                            self.MANIFEST_XML)
-        self._notice_files_dir = os.path.join(install_dir,
-                                              utils.NOTICE_FILES_DIR_PATH)
 
         if not os.path.isfile(self._manifest_file):
             raise RuntimeError(
@@ -197,50 +197,37 @@ class GPLChecker(object):
         """
         logging.info('Starting license check for GPL projects...')
 
-        notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
+        if not self._gpl_projects:
-        if len(notice_files) == 0:
-            raise RuntimeError('No license files found in {}'.format(
-                self._notice_files_dir))
-
-        gpl_projects = []
-        pattern = 'GENERAL PUBLIC LICENSE'
-        for notice_file_path in notice_files:
-            with open(notice_file_path, 'r') as notice_file:
-                if pattern in notice_file.read():
-                    lib_name = os.path.splitext(
-                        os.path.basename(notice_file_path))[0]
-                    gpl_projects.append(lib_name)
-
-        if not gpl_projects:
             logging.info('No GPL projects found.')
             return
 
-        logging.info('GPL projects found: {}'.format(', '.join(gpl_projects)))
+        logging.info('GPL projects found: {}'.format(', '.join(self._gpl_projects)))
 
         module_paths = self._parse_module_paths()
         manifest_projects = self._parse_manifest()
         released_projects = []
         unreleased_projects = []
 
-        for lib in gpl_projects:
+        for name in self._gpl_projects:
-            if lib in module_paths:
+            lib = name if name.endswith('.so') else name + '.so'
-                module_path = module_paths[lib]
+            if lib not in module_paths:
-                revision = self._get_revision(module_path, manifest_projects)
-                if not revision:
-                    raise RuntimeError(
-                        'No project found for {path} in {manifest}'.format(
-                            path=module_path, manifest=self.MANIFEST_XML))
-                revision_exists = self._check_revision_exists(
-                    revision, module_path)
-                if not revision_exists:
-                    unreleased_projects.append((lib, module_path))
-                else:
-                    released_projects.append((lib, module_path))
-            else:
                 raise RuntimeError(
                     'No module path was found for {lib} in {module_paths}'.
                     format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
 
+            module_path = module_paths[lib]
+            revision = self._get_revision(module_path, manifest_projects)
+            if not revision:
+                raise RuntimeError(
+                    'No project found for {path} in {manifest}'.format(
+                        path=module_path, manifest=self.MANIFEST_XML))
+            revision_exists = self._check_revision_exists(
+                revision, module_path)
+            if not revision_exists:
+                unreleased_projects.append((lib, module_path))
+            else:
+                released_projects.append((lib, module_path))
+
         if released_projects:
             logging.info('Released GPL projects: {}'.format(released_projects))
 
@@ -266,6 +253,8 @@ def get_args():
         default='aosp',
         help=('Remote name to fetch and check if the revision of VNDK snapshot '
               'is included in the source to conform GPL license. default=aosp'))
+    parser.add_argument('-m', '--modules', help='list of modules to check',
+                        nargs='+')
     parser.add_argument(
         '-v',
         '--verbose',
@@ -304,7 +293,7 @@ def main():
     utils.fetch_artifact(args.branch, args.build, manifest_pattern,
                          manifest_dest)
 
-    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
+    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, args.modules,
                                  temp_artifact_dir, remote)
     try:
         license_checker.check_gpl_projects()

vndk/snapshot/collect_licenses.py

@@ -36,8 +36,6 @@ LICENSE_KEYWORDS = {
     'NCSA': ('University of Illinois', 'NCSA',),
     'OpenSSL': ('The OpenSSL Project',),
     'Zlib': ('zlib License',),
-}
-RESTRICTED_LICENSE_KEYWORDS = {
     'LGPL-3.0': ('LESSER GENERAL PUBLIC LICENSE', 'Version 3,',),
     'LGPL-2.1': ('LESSER GENERAL PUBLIC LICENSE', 'Version 2.1',),
     'LGPL-2.0': ('GNU LIBRARY GENERAL PUBLIC LICENSE', 'Version 2,',),
@@ -52,12 +50,10 @@ class LicenseCollector(object):
     """ Collect licenses from a VNDK snapshot directory
 
     This is to collect the license_kinds to be used in license modules.
-    It also lists the modules with the restricted licenses.
 
     Initialize the LicenseCollector with a vndk snapshot directory.
     After run() is called, 'license_kinds' will include the licenses found from
     the snapshot directory.
-    'restricted' will have the files that have the restricted licenses.
     """
     def __init__(self, install_dir):
         self._install_dir = install_dir
@@ -66,7 +62,6 @@ class LicenseCollector(object):
         self._paths_to_check = self._paths_to_check + glob.glob(os.path.join(self._install_dir, '*/include'))
 
         self.license_kinds = set()
-        self.restricted = set()
 
     def read_and_check_licenses(self, license_text, license_keywords):
         """ Read the license keywords and check if all keywords are in the file.
@@ -90,17 +85,15 @@ class LicenseCollector(object):
         with open(filepath, 'r') as file_to_check:
             file_string = file_to_check.read()
             self.read_and_check_licenses(file_string, LICENSE_KEYWORDS)
-            if self.read_and_check_licenses(file_string, RESTRICTED_LICENSE_KEYWORDS):
-                self.restricted.add(os.path.basename(filepath))
 
-    def run(self, license_text_path=''):
+    def run(self, module=''):
         """ search licenses in vndk snapshots
 
         Args:
-          license_text_path: path to the license text file to check.
+          module: module name to find the license kind.
-                             If empty, check all license files.
+                  If empty, check all license files.
         """
-        if license_text_path == '':
+        if module == '':
             for path in self._paths_to_check:
                 logging.info('Reading {}'.format(path))
                 for (root, _, files) in os.walk(path):
@@ -108,6 +101,9 @@ class LicenseCollector(object):
                         self.check_licenses(os.path.join(root, f))
             self.license_kinds.update(LICENSE_INCLUDE)
         else:
+            license_text_path = '{notice_dir}/{module}.txt'.format(
+                notice_dir=utils.NOTICE_FILES_DIR_NAME,
+                module=module)
             logging.info('Reading {}'.format(license_text_path))
             self.check_licenses(os.path.join(self._install_dir, utils.COMMON_DIR_PATH, license_text_path))
             if not self.license_kinds:
@@ -143,7 +139,6 @@ def main():
     license_collector = LicenseCollector(install_dir)
     license_collector.run()
     print(sorted(license_collector.license_kinds))
-    print(sorted(license_collector.restricted))
 
 if __name__ == '__main__':
     main()

vndk/snapshot/gen_buildfiles.py

@@ -16,6 +16,7 @@
 #
 
 import argparse
+from collections import defaultdict
 import glob
 import json
 import logging
@@ -68,27 +69,6 @@ class GenBuildFile(object):
         'vndkproduct.libraries.txt',
     ]
 
-    """Some vendor prebuilts reference libprotobuf-cpp-lite.so and
-    libprotobuf-cpp-full.so and expect the 3.0.0-beta3 version.
-    The new version of protobuf will be installed as
-    /vendor/lib64/libprotobuf-cpp-lite-3.9.1.so.  The VNDK doesn't
-    help here because we compile old devices against the current
-    branch and not an old VNDK snapshot.  We need to continue to
-    provide a vendor libprotobuf-cpp-lite.so until all products in
-    the current branch get updated prebuilts or are obsoleted.
-
-    VENDOR_COMPAT is a dictionary that has VNDK versions as keys and
-    the list of (library name string, shared libs list) as values.
-    """
-    VENDOR_COMPAT = {
-        28: [
-            ('libprotobuf-cpp-lite',
-                ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']),
-            ('libprotobuf-cpp-full',
-                ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']),
-        ]
-    }
-
     def __init__(self, install_dir, vndk_version):
         """GenBuildFile constructor.
 
@@ -114,6 +94,10 @@ class GenBuildFile(object):
         self._vndk_product = self._parse_lib_list(
             os.path.basename(self._etc_paths['vndkproduct.libraries.txt']))
         self._modules_with_notice = self._get_modules_with_notice()
+        self._license_in_json = not self._modules_with_notice
+        self._license_kinds_map = defaultdict(set)
+        self._license_texts_map = defaultdict(set)
+        self.modules_with_restricted_lic = set()
 
     def _get_etc_paths(self):
         """Returns a map of relative file paths for each ETC module."""
@@ -164,13 +148,6 @@ class GenBuildFile(object):
         for prebuilt in self.ETC_MODULES:
             prebuilt_buildrules.append(self._gen_etc_prebuilt(prebuilt))
 
-        if self._vndk_version in self.VENDOR_COMPAT:
-            prebuilt_buildrules.append('// Defining prebuilt libraries '
-                        'for the compatibility of old vendor modules')
-            for vendor_compat_lib_info in self.VENDOR_COMPAT[self._vndk_version]:
-                prebuilt_buildrules.append(
-                    self._gen_prebuilt_library_shared(vendor_compat_lib_info))
-
         with open(self._root_bpfile, 'w') as bpfile:
             bpfile.write(self._gen_autogen_msg('/'))
             bpfile.write('\n')
@@ -192,9 +169,14 @@ class GenBuildFile(object):
             bpfile.write(self._gen_autogen_msg('/'))
             bpfile.write('\n')
             bpfile.write(self._gen_license_package())
-            for module in self._modules_with_notice:
+            if self._license_in_json:
-                bpfile.write('\n')
+                for name in self._license_kinds_map:
-                bpfile.write(self._gen_notice_license(module))
+                    bpfile.write('\n')
+                    bpfile.write(self._gen_notice_license(name))
+            else:
+                for module in self._modules_with_notice:
+                    bpfile.write('\n')
+                    bpfile.write(self._gen_notice_license(module))
 
     def generate_android_bp(self):
         """Autogenerates Android.bp."""
@@ -301,17 +283,38 @@ class GenBuildFile(object):
                     ind=self.INDENT,
                     version=self._vndk_version))
 
-    def _get_license_kinds(self, license_text_path=''):
+    def _get_license_kinds(self, module=''):
         """ Returns a set of license kinds
 
         Args:
-          license_text_path: path to the license text file to check.
+          module: module name to find the license kind.
-                             If empty, check all license files.
+                  If empty, check all license files.
         """
+        if self._license_in_json:
+            license_kinds = set()
+            if module == '':
+                # collect all license kinds
+                for kinds in self._license_kinds_map.values():
+                    license_kinds.update(kinds)
+                return license_kinds
+            else:
+                return self._license_kinds_map[module]
+
         license_collector = collect_licenses.LicenseCollector(self._install_dir)
-        license_collector.run(license_text_path)
+        license_collector.run(module)
         return license_collector.license_kinds
 
+    def _get_license_texts(self, module):
+        if self._license_in_json:
+            return {'{notice_dir}/{license_text}'.format(
+                        notice_dir=utils.NOTICE_FILES_DIR_NAME,
+                        license_text=license_text)
+                        for license_text in self._license_texts_map[module]}
+        else:
+            return {'{notice_dir}/{module}.txt'.format(
+                        notice_dir=utils.NOTICE_FILES_DIR_NAME,
+                        module=module)}
+
     def _gen_license(self):
         """ Generates license module.
 
@@ -333,7 +336,7 @@ class GenBuildFile(object):
                     ind=self.INDENT,
                     version=self._vndk_version,
                     license_kinds=license_kinds_string,
-                    notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '*.txt')))
+                    notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '**', '*')))
 
     def _get_versioned_name(self,
                             prebuilt,
@@ -451,29 +454,43 @@ class GenBuildFile(object):
 
     def _gen_notice_license(self, module):
         """Generates a notice license build rule for a given module.
+        When genererating each notice license, collect
+        modules_with_restricted_lic, the list of modules that are under the GPL.
 
         Args:
-          notice: string, module name
+          module: string, module name
         """
-        license_kinds = self._get_license_kinds('{notice_dir}/{module}.txt'.format(
+        def has_restricted_license(license_kinds):
-            notice_dir=utils.NOTICE_FILES_DIR_NAME,
+            for lic in license_kinds:
-            module=module))
+                if 'GPL' in lic:
+                    return True
+            return False
+
+        license_kinds = self._get_license_kinds(module)
+        if has_restricted_license(license_kinds):
+            self.modules_with_restricted_lic.add(module)
         license_kinds_string = ''
         for license_kind in sorted(license_kinds):
             license_kinds_string += '{ind}{ind}"{license_kind}",\n'.format(
                                     ind=self.INDENT, license_kind=license_kind)
+        license_texts = self._get_license_texts(module)
+        license_texts_string = ''
+        for license_text in sorted(license_texts):
+            license_texts_string += '{ind}{ind}"{license_text}",\n'.format(
+                                    ind=self.INDENT, license_text=license_text)
         return ('license {{\n'
                 '{ind}name: "{license_name}",\n'
                 '{ind}license_kinds: [\n'
                 '{license_kinds}'
                 '{ind}],\n'
-                '{ind}license_text: ["{notice_dir}/{module}.txt"],\n'
+                '{ind}license_text: [\n'
+                '{license_texts}'
+                '{ind}],\n'
                 '}}\n'.format(
                     ind=self.INDENT,
                     license_name=self._get_notice_license_name(module),
                     license_kinds=license_kinds_string,
-                    module=module,
+                    license_texts=license_texts_string))
-                    notice_dir=utils.NOTICE_FILES_DIR_NAME))
 
     def _get_notice_license_name(self, module):
         """ Gets a notice license module name for a given module.
@@ -551,6 +568,18 @@ class GenBuildFile(object):
                     return True
             return False
 
+        def get_license_prop(name):
+            """Returns the license prop build rule.
+
+            Args:
+              name: string, name of the module
+            """
+            if name in self._license_kinds_map:
+                return '{ind}licenses: ["{license}"],\n'.format(
+                        ind=self.INDENT,
+                        license=self._get_notice_license_name(name))
+            return ''
+
         def get_notice_file(prebuilts):
             """Returns build rule for notice file (attribute 'licenses').
 
@@ -607,7 +636,7 @@ class GenBuildFile(object):
                             name=name))
 
             def rename_generated_dirs(dirs):
-                # Reame out/soong/.intermedaites to generated-headers for better readability.
+                # Rename out/soong/.intermediates to generated-headers for better readability.
                 return [d.replace(utils.SOONG_INTERMEDIATES_DIR, utils.GENERATED_HEADERS_DIR, 1) for d in dirs]
 
             for src in sorted(src_paths):
@@ -640,6 +669,10 @@ class GenBuildFile(object):
                         'relative_install_path: "{path}",\n').format(
                             ind=self.INDENT,
                             path=props['RelativeInstallPath'])
+                if 'LicenseKinds' in props:
+                    self._license_kinds_map[name].update(props['LicenseKinds'])
+                if 'LicenseTexts' in props:
+                    self._license_texts_map[name].update(props['LicenseTexts'])
 
                 arch_props += ('{ind}{ind}{arch}: {{\n'
                                '{include_dirs}'
@@ -700,8 +733,11 @@ class GenBuildFile(object):
                               vndk_sp=vndk_sp,
                               vndk_private=vndk_private))
 
-        notice = get_notice_file(srcs)
         arch_props = get_arch_props(name, arch, src_paths)
+        if self._license_in_json:
+            license = get_license_prop(name)
+        else:
+            license = get_notice_file(srcs)
 
         binder32bit = ''
         if is_binder32:
@@ -715,7 +751,7 @@ class GenBuildFile(object):
                 '{ind}vendor_available: true,\n'
                 '{product_available}'
                 '{vndk_props}'
-                '{notice}'
+                '{license}'
                 '{arch_props}'
                 '}}\n'.format(
                     ind=self.INDENT,
@@ -725,7 +761,7 @@ class GenBuildFile(object):
                     binder32bit=binder32bit,
                     product_available=product_available,
                     vndk_props=vndk_props,
-                    notice=notice,
+                    license=license,
                     arch_props=arch_props))
 
 
@@ -765,9 +801,11 @@ def main():
     utils.set_logging_config(args.verbose)
 
     buildfile_generator = GenBuildFile(install_dir, vndk_version)
+    # To parse json information, read and generate arch android.bp using
+    # generate_android_bp() first.
+    buildfile_generator.generate_android_bp()
     buildfile_generator.generate_root_android_bp()
     buildfile_generator.generate_common_android_bp()
-    buildfile_generator.generate_android_bp()
 
     logging.info('Done.')
 

vndk/snapshot/update.py

@@ -114,11 +114,13 @@ def gather_notice_files(install_dir):
         notices_dir_per_arch = os.path.join(arch, utils.NOTICE_FILES_DIR_NAME)
         if os.path.isdir(notices_dir_per_arch):
             for notice_file in glob.glob(
-                    '{}/*.txt'.format(notices_dir_per_arch)):
+                    '{}/**'.format(notices_dir_per_arch), recursive=True):
-                if not os.path.isfile(
+                if os.path.isfile(notice_file):
-                        os.path.join(common_notices_dir,
+                    rel_path = os.path.relpath(notice_file, notices_dir_per_arch)
-                                     os.path.basename(notice_file))):
+                    target_path = os.path.join(common_notices_dir, rel_path)
-                    shutil.copy(notice_file, common_notices_dir)
+                    if not os.path.isfile(target_path):
+                        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+                        shutil.copy(notice_file, target_path)
             shutil.rmtree(notices_dir_per_arch)
 
 
@@ -156,15 +158,17 @@ def post_processe_files_if_needed(vndk_version):
 
 
 def update_buildfiles(buildfile_generator):
+    # To parse json information, read and generate arch android.bp using
+    # generate_android_bp() first.
+    logging.info('Generating Android.bp files...')
+    buildfile_generator.generate_android_bp()
+
     logging.info('Generating root Android.bp file...')
     buildfile_generator.generate_root_android_bp()
 
     logging.info('Generating common/Android.bp file...')
     buildfile_generator.generate_common_android_bp()
 
-    logging.info('Generating Android.bp files...')
-    buildfile_generator.generate_android_bp()
-
 def copy_owners(root_dir, install_dir):
     path = os.path.dirname(__file__)
     shutil.copy(os.path.join(root_dir, path, 'OWNERS'), install_dir)
@@ -257,6 +261,7 @@ def run(vndk_version, branch, build_id, local, use_current_branch, remote,
 
         if not local_path and not branch.startswith('android'):
             license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
+                                         buildfile_generator.modules_with_restricted_lic,
                                          temp_artifact_dir, remote)
             check_gpl_license(license_checker)
             logging.info(