diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py index 4451c9b27..d834886d3 100644 --- a/vndk/snapshot/check_gpl_license.py +++ b/vndk/snapshot/check_gpl_license.py @@ -36,14 +36,15 @@ class GPLChecker(object): MANIFEST_XML = utils.MANIFEST_FILE_NAME MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME - def __init__(self, install_dir, android_build_top, temp_artifact_dir, - remote_git): + def __init__(self, install_dir, android_build_top, gpl_projects, + temp_artifact_dir, remote_git): """GPLChecker constructor. Args: install_dir: string, absolute path to the prebuilts/vndk/v{version} directory where the build files will be generated. android_build_top: string, absolute path to ANDROID_BUILD_TOP + gpl_projects: list of strings, names of libraries under GPL temp_artifact_dir: string, temp directory to hold build artifacts fetched from Android Build server. remote_git: string, remote name to fetch and check if the revision of @@ -53,10 +54,9 @@ class GPLChecker(object): self._android_build_top = android_build_top self._install_dir = install_dir self._remote_git = remote_git + self._gpl_projects = gpl_projects self._manifest_file = os.path.join(temp_artifact_dir, self.MANIFEST_XML) - self._notice_files_dir = os.path.join(install_dir, - utils.NOTICE_FILES_DIR_PATH) if not os.path.isfile(self._manifest_file): raise RuntimeError( @@ -197,50 +197,37 @@ class GPLChecker(object): """ logging.info('Starting license check for GPL projects...') - notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) - if len(notice_files) == 0: - raise RuntimeError('No license files found in {}'.format( - self._notice_files_dir)) - - gpl_projects = [] - pattern = 'GENERAL PUBLIC LICENSE' - for notice_file_path in notice_files: - with open(notice_file_path, 'r') as notice_file: - if pattern in notice_file.read(): - lib_name = os.path.splitext( - os.path.basename(notice_file_path))[0] - gpl_projects.append(lib_name) - - if not gpl_projects: + if not self._gpl_projects: logging.info('No GPL projects found.') return - logging.info('GPL projects found: {}'.format(', '.join(gpl_projects))) + logging.info('GPL projects found: {}'.format(', '.join(self._gpl_projects))) module_paths = self._parse_module_paths() manifest_projects = self._parse_manifest() released_projects = [] unreleased_projects = [] - for lib in gpl_projects: - if lib in module_paths: - module_path = module_paths[lib] - revision = self._get_revision(module_path, manifest_projects) - if not revision: - raise RuntimeError( - 'No project found for {path} in {manifest}'.format( - path=module_path, manifest=self.MANIFEST_XML)) - revision_exists = self._check_revision_exists( - revision, module_path) - if not revision_exists: - unreleased_projects.append((lib, module_path)) - else: - released_projects.append((lib, module_path)) - else: + for name in self._gpl_projects: + lib = name if name.endswith('.so') else name + '.so' + if lib not in module_paths: raise RuntimeError( 'No module path was found for {lib} in {module_paths}'. format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) + module_path = module_paths[lib] + revision = self._get_revision(module_path, manifest_projects) + if not revision: + raise RuntimeError( + 'No project found for {path} in {manifest}'.format( + path=module_path, manifest=self.MANIFEST_XML)) + revision_exists = self._check_revision_exists( + revision, module_path) + if not revision_exists: + unreleased_projects.append((lib, module_path)) + else: + released_projects.append((lib, module_path)) + if released_projects: logging.info('Released GPL projects: {}'.format(released_projects)) @@ -266,6 +253,8 @@ def get_args(): default='aosp', help=('Remote name to fetch and check if the revision of VNDK snapshot ' 'is included in the source to conform GPL license. default=aosp')) + parser.add_argument('-m', '--modules', help='list of modules to check', + nargs='+') parser.add_argument( '-v', '--verbose', @@ -304,7 +293,7 @@ def main(): utils.fetch_artifact(args.branch, args.build, manifest_pattern, manifest_dest) - license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, args.modules, temp_artifact_dir, remote) try: license_checker.check_gpl_projects() diff --git a/vndk/snapshot/collect_licenses.py b/vndk/snapshot/collect_licenses.py index 1e98fbd30..f8b0ef29f 100644 --- a/vndk/snapshot/collect_licenses.py +++ b/vndk/snapshot/collect_licenses.py @@ -36,8 +36,6 @@ LICENSE_KEYWORDS = { 'NCSA': ('University of Illinois', 'NCSA',), 'OpenSSL': ('The OpenSSL Project',), 'Zlib': ('zlib License',), -} -RESTRICTED_LICENSE_KEYWORDS = { 'LGPL-3.0': ('LESSER GENERAL PUBLIC LICENSE', 'Version 3,',), 'LGPL-2.1': ('LESSER GENERAL PUBLIC LICENSE', 'Version 2.1',), 'LGPL-2.0': ('GNU LIBRARY GENERAL PUBLIC LICENSE', 'Version 2,',), @@ -52,12 +50,10 @@ class LicenseCollector(object): """ Collect licenses from a VNDK snapshot directory This is to collect the license_kinds to be used in license modules. - It also lists the modules with the restricted licenses. Initialize the LicenseCollector with a vndk snapshot directory. After run() is called, 'license_kinds' will include the licenses found from the snapshot directory. - 'restricted' will have the files that have the restricted licenses. """ def __init__(self, install_dir): self._install_dir = install_dir @@ -66,7 +62,6 @@ class LicenseCollector(object): self._paths_to_check = self._paths_to_check + glob.glob(os.path.join(self._install_dir, '*/include')) self.license_kinds = set() - self.restricted = set() def read_and_check_licenses(self, license_text, license_keywords): """ Read the license keywords and check if all keywords are in the file. @@ -90,17 +85,15 @@ class LicenseCollector(object): with open(filepath, 'r') as file_to_check: file_string = file_to_check.read() self.read_and_check_licenses(file_string, LICENSE_KEYWORDS) - if self.read_and_check_licenses(file_string, RESTRICTED_LICENSE_KEYWORDS): - self.restricted.add(os.path.basename(filepath)) - def run(self, license_text_path=''): + def run(self, module=''): """ search licenses in vndk snapshots Args: - license_text_path: path to the license text file to check. - If empty, check all license files. + module: module name to find the license kind. + If empty, check all license files. """ - if license_text_path == '': + if module == '': for path in self._paths_to_check: logging.info('Reading {}'.format(path)) for (root, _, files) in os.walk(path): @@ -108,6 +101,9 @@ class LicenseCollector(object): self.check_licenses(os.path.join(root, f)) self.license_kinds.update(LICENSE_INCLUDE) else: + license_text_path = '{notice_dir}/{module}.txt'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + module=module) logging.info('Reading {}'.format(license_text_path)) self.check_licenses(os.path.join(self._install_dir, utils.COMMON_DIR_PATH, license_text_path)) if not self.license_kinds: @@ -143,7 +139,6 @@ def main(): license_collector = LicenseCollector(install_dir) license_collector.run() print(sorted(license_collector.license_kinds)) - print(sorted(license_collector.restricted)) if __name__ == '__main__': main() diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index 53ce42f10..0f37b0873 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -16,6 +16,7 @@ # import argparse +from collections import defaultdict import glob import json import logging @@ -68,27 +69,6 @@ class GenBuildFile(object): 'vndkproduct.libraries.txt', ] - """Some vendor prebuilts reference libprotobuf-cpp-lite.so and - libprotobuf-cpp-full.so and expect the 3.0.0-beta3 version. - The new version of protobuf will be installed as - /vendor/lib64/libprotobuf-cpp-lite-3.9.1.so. The VNDK doesn't - help here because we compile old devices against the current - branch and not an old VNDK snapshot. We need to continue to - provide a vendor libprotobuf-cpp-lite.so until all products in - the current branch get updated prebuilts or are obsoleted. - - VENDOR_COMPAT is a dictionary that has VNDK versions as keys and - the list of (library name string, shared libs list) as values. - """ - VENDOR_COMPAT = { - 28: [ - ('libprotobuf-cpp-lite', - ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']), - ('libprotobuf-cpp-full', - ['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']), - ] - } - def __init__(self, install_dir, vndk_version): """GenBuildFile constructor. @@ -114,6 +94,10 @@ class GenBuildFile(object): self._vndk_product = self._parse_lib_list( os.path.basename(self._etc_paths['vndkproduct.libraries.txt'])) self._modules_with_notice = self._get_modules_with_notice() + self._license_in_json = not self._modules_with_notice + self._license_kinds_map = defaultdict(set) + self._license_texts_map = defaultdict(set) + self.modules_with_restricted_lic = set() def _get_etc_paths(self): """Returns a map of relative file paths for each ETC module.""" @@ -164,13 +148,6 @@ class GenBuildFile(object): for prebuilt in self.ETC_MODULES: prebuilt_buildrules.append(self._gen_etc_prebuilt(prebuilt)) - if self._vndk_version in self.VENDOR_COMPAT: - prebuilt_buildrules.append('// Defining prebuilt libraries ' - 'for the compatibility of old vendor modules') - for vendor_compat_lib_info in self.VENDOR_COMPAT[self._vndk_version]: - prebuilt_buildrules.append( - self._gen_prebuilt_library_shared(vendor_compat_lib_info)) - with open(self._root_bpfile, 'w') as bpfile: bpfile.write(self._gen_autogen_msg('/')) bpfile.write('\n') @@ -192,9 +169,14 @@ class GenBuildFile(object): bpfile.write(self._gen_autogen_msg('/')) bpfile.write('\n') bpfile.write(self._gen_license_package()) - for module in self._modules_with_notice: - bpfile.write('\n') - bpfile.write(self._gen_notice_license(module)) + if self._license_in_json: + for name in self._license_kinds_map: + bpfile.write('\n') + bpfile.write(self._gen_notice_license(name)) + else: + for module in self._modules_with_notice: + bpfile.write('\n') + bpfile.write(self._gen_notice_license(module)) def generate_android_bp(self): """Autogenerates Android.bp.""" @@ -301,17 +283,38 @@ class GenBuildFile(object): ind=self.INDENT, version=self._vndk_version)) - def _get_license_kinds(self, license_text_path=''): + def _get_license_kinds(self, module=''): """ Returns a set of license kinds Args: - license_text_path: path to the license text file to check. - If empty, check all license files. + module: module name to find the license kind. + If empty, check all license files. """ + if self._license_in_json: + license_kinds = set() + if module == '': + # collect all license kinds + for kinds in self._license_kinds_map.values(): + license_kinds.update(kinds) + return license_kinds + else: + return self._license_kinds_map[module] + license_collector = collect_licenses.LicenseCollector(self._install_dir) - license_collector.run(license_text_path) + license_collector.run(module) return license_collector.license_kinds + def _get_license_texts(self, module): + if self._license_in_json: + return {'{notice_dir}/{license_text}'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + license_text=license_text) + for license_text in self._license_texts_map[module]} + else: + return {'{notice_dir}/{module}.txt'.format( + notice_dir=utils.NOTICE_FILES_DIR_NAME, + module=module)} + def _gen_license(self): """ Generates license module. @@ -333,7 +336,7 @@ class GenBuildFile(object): ind=self.INDENT, version=self._vndk_version, license_kinds=license_kinds_string, - notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '*.txt'))) + notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '**', '*'))) def _get_versioned_name(self, prebuilt, @@ -451,29 +454,43 @@ class GenBuildFile(object): def _gen_notice_license(self, module): """Generates a notice license build rule for a given module. + When genererating each notice license, collect + modules_with_restricted_lic, the list of modules that are under the GPL. Args: - notice: string, module name + module: string, module name """ - license_kinds = self._get_license_kinds('{notice_dir}/{module}.txt'.format( - notice_dir=utils.NOTICE_FILES_DIR_NAME, - module=module)) + def has_restricted_license(license_kinds): + for lic in license_kinds: + if 'GPL' in lic: + return True + return False + + license_kinds = self._get_license_kinds(module) + if has_restricted_license(license_kinds): + self.modules_with_restricted_lic.add(module) license_kinds_string = '' for license_kind in sorted(license_kinds): license_kinds_string += '{ind}{ind}"{license_kind}",\n'.format( ind=self.INDENT, license_kind=license_kind) + license_texts = self._get_license_texts(module) + license_texts_string = '' + for license_text in sorted(license_texts): + license_texts_string += '{ind}{ind}"{license_text}",\n'.format( + ind=self.INDENT, license_text=license_text) return ('license {{\n' '{ind}name: "{license_name}",\n' '{ind}license_kinds: [\n' '{license_kinds}' '{ind}],\n' - '{ind}license_text: ["{notice_dir}/{module}.txt"],\n' + '{ind}license_text: [\n' + '{license_texts}' + '{ind}],\n' '}}\n'.format( ind=self.INDENT, license_name=self._get_notice_license_name(module), license_kinds=license_kinds_string, - module=module, - notice_dir=utils.NOTICE_FILES_DIR_NAME)) + license_texts=license_texts_string)) def _get_notice_license_name(self, module): """ Gets a notice license module name for a given module. @@ -551,6 +568,18 @@ class GenBuildFile(object): return True return False + def get_license_prop(name): + """Returns the license prop build rule. + + Args: + name: string, name of the module + """ + if name in self._license_kinds_map: + return '{ind}licenses: ["{license}"],\n'.format( + ind=self.INDENT, + license=self._get_notice_license_name(name)) + return '' + def get_notice_file(prebuilts): """Returns build rule for notice file (attribute 'licenses'). @@ -607,7 +636,7 @@ class GenBuildFile(object): name=name)) def rename_generated_dirs(dirs): - # Reame out/soong/.intermedaites to generated-headers for better readability. + # Rename out/soong/.intermediates to generated-headers for better readability. return [d.replace(utils.SOONG_INTERMEDIATES_DIR, utils.GENERATED_HEADERS_DIR, 1) for d in dirs] for src in sorted(src_paths): @@ -640,6 +669,10 @@ class GenBuildFile(object): 'relative_install_path: "{path}",\n').format( ind=self.INDENT, path=props['RelativeInstallPath']) + if 'LicenseKinds' in props: + self._license_kinds_map[name].update(props['LicenseKinds']) + if 'LicenseTexts' in props: + self._license_texts_map[name].update(props['LicenseTexts']) arch_props += ('{ind}{ind}{arch}: {{\n' '{include_dirs}' @@ -700,8 +733,11 @@ class GenBuildFile(object): vndk_sp=vndk_sp, vndk_private=vndk_private)) - notice = get_notice_file(srcs) arch_props = get_arch_props(name, arch, src_paths) + if self._license_in_json: + license = get_license_prop(name) + else: + license = get_notice_file(srcs) binder32bit = '' if is_binder32: @@ -715,7 +751,7 @@ class GenBuildFile(object): '{ind}vendor_available: true,\n' '{product_available}' '{vndk_props}' - '{notice}' + '{license}' '{arch_props}' '}}\n'.format( ind=self.INDENT, @@ -725,7 +761,7 @@ class GenBuildFile(object): binder32bit=binder32bit, product_available=product_available, vndk_props=vndk_props, - notice=notice, + license=license, arch_props=arch_props)) @@ -765,9 +801,11 @@ def main(): utils.set_logging_config(args.verbose) buildfile_generator = GenBuildFile(install_dir, vndk_version) + # To parse json information, read and generate arch android.bp using + # generate_android_bp() first. + buildfile_generator.generate_android_bp() buildfile_generator.generate_root_android_bp() buildfile_generator.generate_common_android_bp() - buildfile_generator.generate_android_bp() logging.info('Done.') diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index 071dfb843..cd0fcfd02 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -114,11 +114,13 @@ def gather_notice_files(install_dir): notices_dir_per_arch = os.path.join(arch, utils.NOTICE_FILES_DIR_NAME) if os.path.isdir(notices_dir_per_arch): for notice_file in glob.glob( - '{}/*.txt'.format(notices_dir_per_arch)): - if not os.path.isfile( - os.path.join(common_notices_dir, - os.path.basename(notice_file))): - shutil.copy(notice_file, common_notices_dir) + '{}/**'.format(notices_dir_per_arch), recursive=True): + if os.path.isfile(notice_file): + rel_path = os.path.relpath(notice_file, notices_dir_per_arch) + target_path = os.path.join(common_notices_dir, rel_path) + if not os.path.isfile(target_path): + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.copy(notice_file, target_path) shutil.rmtree(notices_dir_per_arch) @@ -156,15 +158,17 @@ def post_processe_files_if_needed(vndk_version): def update_buildfiles(buildfile_generator): + # To parse json information, read and generate arch android.bp using + # generate_android_bp() first. + logging.info('Generating Android.bp files...') + buildfile_generator.generate_android_bp() + logging.info('Generating root Android.bp file...') buildfile_generator.generate_root_android_bp() logging.info('Generating common/Android.bp file...') buildfile_generator.generate_common_android_bp() - logging.info('Generating Android.bp files...') - buildfile_generator.generate_android_bp() - def copy_owners(root_dir, install_dir): path = os.path.dirname(__file__) shutil.copy(os.path.join(root_dir, path, 'OWNERS'), install_dir) @@ -257,6 +261,7 @@ def run(vndk_version, branch, build_id, local, use_current_branch, remote, if not local_path and not branch.startswith('android'): license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + buildfile_generator.modules_with_restricted_lic, temp_artifact_dir, remote) check_gpl_license(license_checker) logging.info(