From 9e48f54e06a70cb279ab5f278e0b8e3ba7f9f4f4 Mon Sep 17 00:00:00 2001 From: Justin Yun Date: Mon, 17 Apr 2023 12:23:01 +0900 Subject: [PATCH] VNDK snapshot tool generates license information from json VNDK snapshot zip file includes license information in json files. Instead of parsing the license text files, read the information from the json files. VNDK snapshot v33 and older do not include this information and must parse the license text files as before. Bug: 277317599 Test: generate snapshots with/without json information && m nothing Change-Id: Ibdbc571c9effe6c17f5d82a7a0894ad2d800711f --- vndk/snapshot/check_gpl_license.py | 61 ++++++------- vndk/snapshot/collect_licenses.py | 19 ++-- vndk/snapshot/gen_buildfiles.py | 134 ++++++++++++++++++----------- vndk/snapshot/update.py | 21 +++-- 4 files changed, 131 insertions(+), 104 deletions(-) diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py index 4451c9b27..d834886d3 100644 --- a/vndk/snapshot/check_gpl_license.py +++ b/vndk/snapshot/check_gpl_license.py @@ -36,14 +36,15 @@ class GPLChecker(object): MANIFEST_XML = utils.MANIFEST_FILE_NAME MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME - def __init__(self, install_dir, android_build_top, temp_artifact_dir, - remote_git): + def __init__(self, install_dir, android_build_top, gpl_projects, + temp_artifact_dir, remote_git): """GPLChecker constructor. Args: install_dir: string, absolute path to the prebuilts/vndk/v{version} directory where the build files will be generated. android_build_top: string, absolute path to ANDROID_BUILD_TOP + gpl_projects: list of strings, names of libraries under GPL temp_artifact_dir: string, temp directory to hold build artifacts fetched from Android Build server. remote_git: string, remote name to fetch and check if the revision of @@ -53,10 +54,9 @@ class GPLChecker(object): self._android_build_top = android_build_top self._install_dir = install_dir self._remote_git = remote_git + self._gpl_projects = gpl_projects self._manifest_file = os.path.join(temp_artifact_dir, self.MANIFEST_XML) - self._notice_files_dir = os.path.join(install_dir, - utils.NOTICE_FILES_DIR_PATH) if not os.path.isfile(self._manifest_file): raise RuntimeError( @@ -197,50 +197,37 @@ class GPLChecker(object): """ logging.info('Starting license check for GPL projects...') - notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) - if len(notice_files) == 0: - raise RuntimeError('No license files found in {}'.format( - self._notice_files_dir)) - - gpl_projects = [] - pattern = 'GENERAL PUBLIC LICENSE' - for notice_file_path in notice_files: - with open(notice_file_path, 'r') as notice_file: - if pattern in notice_file.read(): - lib_name = os.path.splitext( - os.path.basename(notice_file_path))[0] - gpl_projects.append(lib_name) - - if not gpl_projects: + if not self._gpl_projects: logging.info('No GPL projects found.') return - logging.info('GPL projects found: {}'.format(', '.join(gpl_projects))) + logging.info('GPL projects found: {}'.format(', '.join(self._gpl_projects))) module_paths = self._parse_module_paths() manifest_projects = self._parse_manifest() released_projects = [] unreleased_projects = [] - for lib in gpl_projects: - if lib in module_paths: - module_path = module_paths[lib] - revision = self._get_revision(module_path, manifest_projects) - if not revision: - raise RuntimeError( - 'No project found for {path} in {manifest}'.format( - path=module_path, manifest=self.MANIFEST_XML)) - revision_exists = self._check_revision_exists( - revision, module_path) - if not revision_exists: - unreleased_projects.append((lib, module_path)) - else: - released_projects.append((lib, module_path)) - else: + for name in self._gpl_projects: + lib = name if name.endswith('.so') else name + '.so' + if lib not in module_paths: raise RuntimeError( 'No module path was found for {lib} in {module_paths}'. format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) + module_path = module_paths[lib] + revision = self._get_revision(module_path, manifest_projects) + if not revision: + raise RuntimeError( + 'No project found for {path} in {manifest}'.format( + path=module_path, manifest=self.MANIFEST_XML)) + revision_exists = self._check_revision_exists( + revision, module_path) + if not revision_exists: + unreleased_projects.append((lib, module_path)) + else: + released_projects.append((lib, module_path)) + if released_projects: logging.info('Released GPL projects: {}'.format(released_projects)) @@ -266,6 +253,8 @@ def get_args(): default='aosp', help=('Remote name to fetch and check if the revision of VNDK snapshot ' 'is included in the source to conform GPL license. default=aosp')) + parser.add_argument('-m', '--modules', help='list of modules to check', + nargs='+') parser.add_argument( '-v', '--verbose', @@ -304,7 +293,7 @@ def main(): utils.fetch_artifact(args.branch, args.build, manifest_pattern, manifest_dest) - license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, args.modules, temp_artifact_dir, remote) try: license_checker.check_gpl_projects() diff --git a/vndk/snapshot/collect_licenses.py b/vndk/snapshot/collect_licenses.py index 1e98fbd30..f8b0ef29f 100644 --- a/vndk/snapshot/collect_licenses.py +++ b/vndk/snapshot/collect_licenses.py @@ -36,8 +36,6 @@ LICENSE_KEYWORDS = { 'NCSA': ('University of Illinois', 'NCSA',), 'OpenSSL': ('The OpenSSL Project',), 'Zlib': ('zlib License',), -} -RESTRICTED_LICENSE_KEYWORDS = { 'LGPL-3.0': ('LESSER GENERAL PUBLIC LICENSE', 'Version 3,',), 'LGPL-2.1': ('LESSER GENERAL PUBLIC LICENSE', 'Version 2.1',), 'LGPL-2.0': ('GNU LIBRARY GENERAL PUBLIC LICENSE', 'Version 2,',), @@ -52,12 +50,10 @@ class LicenseCollector(object): """ Collect licenses from a VNDK snapshot directory This is to collect the license_kinds to be used in license modules. - It also lists the modules with the restricted licenses. Initialize the LicenseCollector with a vndk snapshot directory. After run() is called, 'license_kinds' will include the licenses found from the snapshot directory. - 'restricted' will have the files that have the restricted licenses. """ def __init__(self, install_dir): self._install_dir = install_dir @@ -66,7 +62,6 @@ class LicenseCollector(object): self._paths_to_check = self._paths_to_check + glob.glob(os.path.join(self._install_dir, '*/include')) self.license_kinds = set() - self.restricted = set() def read_and_check_licenses(self, license_text, license_keywords): """ Read the license keywords and check if all keywords are in the file. @@ -90,17 +85,15 @@ class LicenseCollector(object): with open(filepath, 'r') as file_to_check: file_string = file_to_check.read() self.read_and_check_licenses(file_string, LICENSE_KEYWORDS) - if self.read_and_check_licenses(file_string, RESTRICTED_LICENSE_KEYWORDS): - self.restricted.add(os.path.basename(filepath)) - def run(self, license_text_path=''): + def run(self, module=''): """ search licenses in vndk snapshots Args: - license_text_path: path to the license text file to check. - If empty, check all license files. + module: module name to find the license kind. + If empty, check all license files. """ - if license_text_path == '': + if module == '': for path in self._paths_to_check: logging.info('Reading {}'.format(path)) for (root, _, files) in os.walk(path): @@ -108,6 +101,9 @@ class LicenseCollector(object): self.check_licenses(os.path.join(root, f)) self.license_kinds.update(LICENSE_INCLUDE) else: + license_text_path = '{notice_dir}/{module}.txt'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + module=module) logging.info('Reading {}'.format(license_text_path)) self.check_licenses(os.path.join(self._install_dir, utils.COMMON_DIR_PATH, license_text_path)) if not self.license_kinds: @@ -143,7 +139,6 @@ def main(): license_collector = LicenseCollector(install_dir) license_collector.run() print(sorted(license_collector.license_kinds)) - print(sorted(license_collector.restricted)) if __name__ == '__main__': main() diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index 53ce42f10..0f37b0873 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -16,6 +16,7 @@ # import argparse +from collections import defaultdict import glob import json import logging @@ -68,27 +69,6 @@ class GenBuildFile(object): 'vndkproduct.libraries.txt', ] - """Some vendor prebuilts reference libprotobuf-cpp-lite.so and - libprotobuf-cpp-full.so and expect the 3.0.0-beta3 version. - The new version of protobuf will be installed as - /vendor/lib64/libprotobuf-cpp-lite-3.9.1.so. The VNDK doesn't - help here because we compile old devices against the current - branch and not an old VNDK snapshot. We need to continue to - provide a vendor libprotobuf-cpp-lite.so until all products in - the current branch get updated prebuilts or are obsoleted. - - VENDOR_COMPAT is a dictionary that has VNDK versions as keys and - the list of (library name string, shared libs list) as values. - """ - VENDOR_COMPAT = { - 28: [ - ('libprotobuf-cpp-lite', - ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']), - ('libprotobuf-cpp-full', - ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']), - ] - } - def __init__(self, install_dir, vndk_version): """GenBuildFile constructor. @@ -114,6 +94,10 @@ class GenBuildFile(object): self._vndk_product = self._parse_lib_list( os.path.basename(self._etc_paths['vndkproduct.libraries.txt'])) self._modules_with_notice = self._get_modules_with_notice() + self._license_in_json = not self._modules_with_notice + self._license_kinds_map = defaultdict(set) + self._license_texts_map = defaultdict(set) + self.modules_with_restricted_lic = set() def _get_etc_paths(self): """Returns a map of relative file paths for each ETC module.""" @@ -164,13 +148,6 @@ class GenBuildFile(object): for prebuilt in self.ETC_MODULES: prebuilt_buildrules.append(self._gen_etc_prebuilt(prebuilt)) - if self._vndk_version in self.VENDOR_COMPAT: - prebuilt_buildrules.append('// Defining prebuilt libraries ' - 'for the compatibility of old vendor modules') - for vendor_compat_lib_info in self.VENDOR_COMPAT[self._vndk_version]: - prebuilt_buildrules.append( - self._gen_prebuilt_library_shared(vendor_compat_lib_info)) - with open(self._root_bpfile, 'w') as bpfile: bpfile.write(self._gen_autogen_msg('/')) bpfile.write('\n') @@ -192,9 +169,14 @@ class GenBuildFile(object): bpfile.write(self._gen_autogen_msg('/')) bpfile.write('\n') bpfile.write(self._gen_license_package()) - for module in self._modules_with_notice: - bpfile.write('\n') - bpfile.write(self._gen_notice_license(module)) + if self._license_in_json: + for name in self._license_kinds_map: + bpfile.write('\n') + bpfile.write(self._gen_notice_license(name)) + else: + for module in self._modules_with_notice: + bpfile.write('\n') + bpfile.write(self._gen_notice_license(module)) def generate_android_bp(self): """Autogenerates Android.bp.""" @@ -301,17 +283,38 @@ class GenBuildFile(object): ind=self.INDENT, version=self._vndk_version)) - def _get_license_kinds(self, license_text_path=''): + def _get_license_kinds(self, module=''): """ Returns a set of license kinds Args: - license_text_path: path to the license text file to check. - If empty, check all license files. + module: module name to find the license kind. + If empty, check all license files. """ + if self._license_in_json: + license_kinds = set() + if module == '': + # collect all license kinds + for kinds in self._license_kinds_map.values(): + license_kinds.update(kinds) + return license_kinds + else: + return self._license_kinds_map[module] + license_collector = collect_licenses.LicenseCollector(self._install_dir) - license_collector.run(license_text_path) + license_collector.run(module) return license_collector.license_kinds + def _get_license_texts(self, module): + if self._license_in_json: + return {'{notice_dir}/{license_text}'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + license_text=license_text) + for license_text in self._license_texts_map[module]} + else: + return {'{notice_dir}/{module}.txt'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + module=module)} + def _gen_license(self): """ Generates license module. @@ -333,7 +336,7 @@ class GenBuildFile(object): ind=self.INDENT, version=self._vndk_version, license_kinds=license_kinds_string, - notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '*.txt'))) + notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '**', '*'))) def _get_versioned_name(self, prebuilt, @@ -451,29 +454,43 @@ class GenBuildFile(object): def _gen_notice_license(self, module): """Generates a notice license build rule for a given module. + When genererating each notice license, collect + modules_with_restricted_lic, the list of modules that are under the GPL. Args: - notice: string, module name + module: string, module name """ - license_kinds = self._get_license_kinds('{notice_dir}/{module}.txt'.format( - notice_dir=utils.NOTICE_FILES_DIR_NAME, - module=module)) + def has_restricted_license(license_kinds): + for lic in license_kinds: + if 'GPL' in lic: + return True + return False + + license_kinds = self._get_license_kinds(module) + if has_restricted_license(license_kinds): + self.modules_with_restricted_lic.add(module) license_kinds_string = '' for license_kind in sorted(license_kinds): license_kinds_string += '{ind}{ind}"{license_kind}",\n'.format( ind=self.INDENT, license_kind=license_kind) + license_texts = self._get_license_texts(module) + license_texts_string = '' + for license_text in sorted(license_texts): + license_texts_string += '{ind}{ind}"{license_text}",\n'.format( + ind=self.INDENT, license_text=license_text) return ('license {{\n' '{ind}name: "{license_name}",\n' '{ind}license_kinds: [\n' '{license_kinds}' '{ind}],\n' - '{ind}license_text: ["{notice_dir}/{module}.txt"],\n' + '{ind}license_text: [\n' + '{license_texts}' + '{ind}],\n' '}}\n'.format( ind=self.INDENT, license_name=self._get_notice_license_name(module), license_kinds=license_kinds_string, - module=module, - notice_dir=utils.NOTICE_FILES_DIR_NAME)) + license_texts=license_texts_string)) def _get_notice_license_name(self, module): """ Gets a notice license module name for a given module. @@ -551,6 +568,18 @@ class GenBuildFile(object): return True return False + def get_license_prop(name): + """Returns the license prop build rule. + + Args: + name: string, name of the module + """ + if name in self._license_kinds_map: + return '{ind}licenses: ["{license}"],\n'.format( + ind=self.INDENT, + license=self._get_notice_license_name(name)) + return '' + def get_notice_file(prebuilts): """Returns build rule for notice file (attribute 'licenses'). @@ -607,7 +636,7 @@ class GenBuildFile(object): name=name)) def rename_generated_dirs(dirs): - # Reame out/soong/.intermedaites to generated-headers for better readability. + # Rename out/soong/.intermediates to generated-headers for better readability. return [d.replace(utils.SOONG_INTERMEDIATES_DIR, utils.GENERATED_HEADERS_DIR, 1) for d in dirs] for src in sorted(src_paths): @@ -640,6 +669,10 @@ class GenBuildFile(object): 'relative_install_path: "{path}",\n').format( ind=self.INDENT, path=props['RelativeInstallPath']) + if 'LicenseKinds' in props: + self._license_kinds_map[name].update(props['LicenseKinds']) + if 'LicenseTexts' in props: + self._license_texts_map[name].update(props['LicenseTexts']) arch_props += ('{ind}{ind}{arch}: {{\n' '{include_dirs}' @@ -700,8 +733,11 @@ class GenBuildFile(object): vndk_sp=vndk_sp, vndk_private=vndk_private)) - notice = get_notice_file(srcs) arch_props = get_arch_props(name, arch, src_paths) + if self._license_in_json: + license = get_license_prop(name) + else: + license = get_notice_file(srcs) binder32bit = '' if is_binder32: @@ -715,7 +751,7 @@ class GenBuildFile(object): '{ind}vendor_available: true,\n' '{product_available}' '{vndk_props}' - '{notice}' + '{license}' '{arch_props}' '}}\n'.format( ind=self.INDENT, @@ -725,7 +761,7 @@ class GenBuildFile(object): binder32bit=binder32bit, product_available=product_available, vndk_props=vndk_props, - notice=notice, + license=license, arch_props=arch_props)) @@ -765,9 +801,11 @@ def main(): utils.set_logging_config(args.verbose) buildfile_generator = GenBuildFile(install_dir, vndk_version) + # To parse json information, read and generate arch android.bp using + # generate_android_bp() first. + buildfile_generator.generate_android_bp() buildfile_generator.generate_root_android_bp() buildfile_generator.generate_common_android_bp() - buildfile_generator.generate_android_bp() logging.info('Done.') diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index 071dfb843..cd0fcfd02 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -114,11 +114,13 @@ def gather_notice_files(install_dir): notices_dir_per_arch = os.path.join(arch, utils.NOTICE_FILES_DIR_NAME) if os.path.isdir(notices_dir_per_arch): for notice_file in glob.glob( - '{}/*.txt'.format(notices_dir_per_arch)): - if not os.path.isfile( - os.path.join(common_notices_dir, - os.path.basename(notice_file))): - shutil.copy(notice_file, common_notices_dir) + '{}/**'.format(notices_dir_per_arch), recursive=True): + if os.path.isfile(notice_file): + rel_path = os.path.relpath(notice_file, notices_dir_per_arch) + target_path = os.path.join(common_notices_dir, rel_path) + if not os.path.isfile(target_path): + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.copy(notice_file, target_path) shutil.rmtree(notices_dir_per_arch) @@ -156,15 +158,17 @@ def post_processe_files_if_needed(vndk_version): def update_buildfiles(buildfile_generator): + # To parse json information, read and generate arch android.bp using + # generate_android_bp() first. + logging.info('Generating Android.bp files...') + buildfile_generator.generate_android_bp() + logging.info('Generating root Android.bp file...') buildfile_generator.generate_root_android_bp() logging.info('Generating common/Android.bp file...') buildfile_generator.generate_common_android_bp() - logging.info('Generating Android.bp files...') - buildfile_generator.generate_android_bp() - def copy_owners(root_dir, install_dir): path = os.path.dirname(__file__) shutil.copy(os.path.join(root_dir, path, 'OWNERS'), install_dir) @@ -257,6 +261,7 @@ def run(vndk_version, branch, build_id, local, use_current_branch, remote, if not local_path and not branch.startswith('android'): license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + buildfile_generator.modules_with_restricted_lic, temp_artifact_dir, remote) check_gpl_license(license_checker) logging.info(