LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "VNDK snapshot tool generates license information from json" am: c973f68913 am: b7e7077c5e

Browse Source 
Original change: https://android-review.googlesource.com/c/platform/development/+/2539710

Change-Id: I98d372acda2b5eab1a59e6eef58fb96f39668a6e
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Treehugger Robot
2023-04-18 06:52:57 +00:00
committed by Automerger Merge Worker
parent e139706c90 b7e7077c5e
commit d8a0e2be23
4 changed files with 131 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
vndk/snapshot/check_gpl_license.py
View File

@@ -36,14 +36,15 @@ class GPLChecker(object):
MANIFEST_XML = utils.MANIFEST_FILE_NAME
MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
def __init__(self, install_dir, android_build_top, temp_artifact_dir,
remote_git):
def __init__(self, install_dir, android_build_top, gpl_projects,
temp_artifact_dir, remote_git):
"""GPLChecker constructor.
Args:
install_dir: string, absolute path to the prebuilts/vndk/v{version}
directory where the build files will be generated.
android_build_top: string, absolute path to ANDROID_BUILD_TOP
gpl_projects: list of strings, names of libraries under GPL
temp_artifact_dir: string, temp directory to hold build artifacts
fetched from Android Build server.
remote_git: string, remote name to fetch and check if the revision of
@@ -53,10 +54,9 @@ class GPLChecker(object):
self._android_build_top = android_build_top
self._install_dir = install_dir
self._remote_git = remote_git
self._gpl_projects = gpl_projects
self._manifest_file = os.path.join(temp_artifact_dir,
self.MANIFEST_XML)
self._notice_files_dir = os.path.join(install_dir,
utils.NOTICE_FILES_DIR_PATH)
if not os.path.isfile(self._manifest_file):
raise RuntimeError(
@@ -197,50 +197,37 @@ class GPLChecker(object):
"""
logging.info('Starting license check for GPL projects...')
notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
if len(notice_files) == 0:
raise RuntimeError('No license files found in {}'.format(
self._notice_files_dir))
gpl_projects = []
pattern = 'GENERAL PUBLIC LICENSE'
for notice_file_path in notice_files:
with open(notice_file_path, 'r') as notice_file:
if pattern in notice_file.read():
lib_name = os.path.splitext(
os.path.basename(notice_file_path))[0]
gpl_projects.append(lib_name)
if not gpl_projects:
if not self._gpl_projects:
logging.info('No GPL projects found.')
return
logging.info('GPL projects found: {}'.format(', '.join(gpl_projects)))
logging.info('GPL projects found: {}'.format(', '.join(self._gpl_projects)))
module_paths = self._parse_module_paths()
manifest_projects = self._parse_manifest()
released_projects = []
unreleased_projects = []
for lib in gpl_projects:
if lib in module_paths:
module_path = module_paths[lib]
revision = self._get_revision(module_path, manifest_projects)
if not revision:
raise RuntimeError(
'No project found for {path} in {manifest}'.format(
path=module_path, manifest=self.MANIFEST_XML))
revision_exists = self._check_revision_exists(
revision, module_path)
if not revision_exists:
unreleased_projects.append((lib, module_path))
else:
released_projects.append((lib, module_path))
else:
for name in self._gpl_projects:
lib = name if name.endswith('.so') else name + '.so'
if lib not in module_paths:
raise RuntimeError(
'No module path was found for {lib} in {module_paths}'.
format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
module_path = module_paths[lib]
revision = self._get_revision(module_path, manifest_projects)
if not revision:
raise RuntimeError(
'No project found for {path} in {manifest}'.format(
path=module_path, manifest=self.MANIFEST_XML))
revision_exists = self._check_revision_exists(
revision, module_path)
if not revision_exists:
unreleased_projects.append((lib, module_path))
else:
released_projects.append((lib, module_path))
if released_projects:
logging.info('Released GPL projects: {}'.format(released_projects))
@@ -266,6 +253,8 @@ def get_args():
default='aosp',
help=('Remote name to fetch and check if the revision of VNDK snapshot '
'is included in the source to conform GPL license. default=aosp'))
parser.add_argument('-m', '--modules', help='list of modules to check',
nargs='+')
parser.add_argument(
'-v',
'--verbose',
@@ -304,7 +293,7 @@ def main():
utils.fetch_artifact(args.branch, args.build, manifest_pattern,
manifest_dest)
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, args.modules,
temp_artifact_dir, remote)
try:
license_checker.check_gpl_projects()

19
vndk/snapshot/collect_licenses.py
View File

@@ -36,8 +36,6 @@ LICENSE_KEYWORDS = {
'NCSA': ('University of Illinois', 'NCSA',),
'OpenSSL': ('The OpenSSL Project',),
'Zlib': ('zlib License',),
}
RESTRICTED_LICENSE_KEYWORDS = {
'LGPL-3.0': ('LESSER GENERAL PUBLIC LICENSE', 'Version 3,',),
'LGPL-2.1': ('LESSER GENERAL PUBLIC LICENSE', 'Version 2.1',),
'LGPL-2.0': ('GNU LIBRARY GENERAL PUBLIC LICENSE', 'Version 2,',),
@@ -52,12 +50,10 @@ class LicenseCollector(object):
""" Collect licenses from a VNDK snapshot directory
This is to collect the license_kinds to be used in license modules.
It also lists the modules with the restricted licenses.
Initialize the LicenseCollector with a vndk snapshot directory.
After run() is called, 'license_kinds' will include the licenses found from
the snapshot directory.
'restricted' will have the files that have the restricted licenses.
"""
def __init__(self, install_dir):
self._install_dir = install_dir
@@ -66,7 +62,6 @@ class LicenseCollector(object):
self._paths_to_check = self._paths_to_check + glob.glob(os.path.join(self._install_dir, '*/include'))
self.license_kinds = set()
self.restricted = set()
def read_and_check_licenses(self, license_text, license_keywords):
""" Read the license keywords and check if all keywords are in the file.
@@ -90,17 +85,15 @@ class LicenseCollector(object):
with open(filepath, 'r') as file_to_check:
file_string = file_to_check.read()
self.read_and_check_licenses(file_string, LICENSE_KEYWORDS)
if self.read_and_check_licenses(file_string, RESTRICTED_LICENSE_KEYWORDS):
self.restricted.add(os.path.basename(filepath))
def run(self, license_text_path=''):
def run(self, module=''):
""" search licenses in vndk snapshots
Args:
license_text_path: path to the license text file to check.
If empty, check all license files.
module: module name to find the license kind.
If empty, check all license files.
"""
if license_text_path == '':
if module == '':
for path in self._paths_to_check:
logging.info('Reading {}'.format(path))
for (root, _, files) in os.walk(path):
@@ -108,6 +101,9 @@ class LicenseCollector(object):
self.check_licenses(os.path.join(root, f))
self.license_kinds.update(LICENSE_INCLUDE)
else:
license_text_path = '{notice_dir}/{module}.txt'.format(
notice_dir=utils.NOTICE_FILES_DIR_NAME,
module=module)
logging.info('Reading {}'.format(license_text_path))
self.check_licenses(os.path.join(self._install_dir, utils.COMMON_DIR_PATH, license_text_path))
if not self.license_kinds:
@@ -143,7 +139,6 @@ def main():
license_collector = LicenseCollector(install_dir)
license_collector.run()
print(sorted(license_collector.license_kinds))
print(sorted(license_collector.restricted))
if __name__ == '__main__':
main()

134
vndk/snapshot/gen_buildfiles.py
View File

@@ -16,6 +16,7 @@
#
import argparse
from collections import defaultdict
import glob
import json
import logging
@@ -68,27 +69,6 @@ class GenBuildFile(object):
'vndkproduct.libraries.txt',
]
"""Some vendor prebuilts reference libprotobuf-cpp-lite.so and
libprotobuf-cpp-full.so and expect the 3.0.0-beta3 version.
The new version of protobuf will be installed as
/vendor/lib64/libprotobuf-cpp-lite-3.9.1.so. The VNDK doesn't
help here because we compile old devices against the current
branch and not an old VNDK snapshot. We need to continue to
provide a vendor libprotobuf-cpp-lite.so until all products in
the current branch get updated prebuilts or are obsoleted.
VENDOR_COMPAT is a dictionary that has VNDK versions as keys and
the list of (library name string, shared libs list) as values.
"""
VENDOR_COMPAT = {
28: [
('libprotobuf-cpp-lite',
['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']),
('libprotobuf-cpp-full',
['libc++', 'libc', 'libdl', 'liblog', 'libm', 'libz']),
]
}
def __init__(self, install_dir, vndk_version):
"""GenBuildFile constructor.
@@ -114,6 +94,10 @@ class GenBuildFile(object):
self._vndk_product = self._parse_lib_list(
os.path.basename(self._etc_paths['vndkproduct.libraries.txt']))
self._modules_with_notice = self._get_modules_with_notice()
self._license_in_json = not self._modules_with_notice
self._license_kinds_map = defaultdict(set)
self._license_texts_map = defaultdict(set)
self.modules_with_restricted_lic = set()
def _get_etc_paths(self):
"""Returns a map of relative file paths for each ETC module."""
@@ -164,13 +148,6 @@ class GenBuildFile(object):
for prebuilt in self.ETC_MODULES:
prebuilt_buildrules.append(self._gen_etc_prebuilt(prebuilt))
if self._vndk_version in self.VENDOR_COMPAT:
prebuilt_buildrules.append('// Defining prebuilt libraries '
'for the compatibility of old vendor modules')
for vendor_compat_lib_info in self.VENDOR_COMPAT[self._vndk_version]:
prebuilt_buildrules.append(
self._gen_prebuilt_library_shared(vendor_compat_lib_info))
with open(self._root_bpfile, 'w') as bpfile:
bpfile.write(self._gen_autogen_msg('/'))
bpfile.write('\n')
@@ -192,9 +169,14 @@ class GenBuildFile(object):
bpfile.write(self._gen_autogen_msg('/'))
bpfile.write('\n')
bpfile.write(self._gen_license_package())
for module in self._modules_with_notice:
bpfile.write('\n')
bpfile.write(self._gen_notice_license(module))
if self._license_in_json:
for name in self._license_kinds_map:
bpfile.write('\n')
bpfile.write(self._gen_notice_license(name))
else:
for module in self._modules_with_notice:
bpfile.write('\n')
bpfile.write(self._gen_notice_license(module))
def generate_android_bp(self):
"""Autogenerates Android.bp."""
@@ -301,17 +283,38 @@ class GenBuildFile(object):
ind=self.INDENT,
version=self._vndk_version))
def _get_license_kinds(self, license_text_path=''):
def _get_license_kinds(self, module=''):
""" Returns a set of license kinds
Args:
license_text_path: path to the license text file to check.
If empty, check all license files.
module: module name to find the license kind.
If empty, check all license files.
"""
if self._license_in_json:
license_kinds = set()
if module == '':
# collect all license kinds
for kinds in self._license_kinds_map.values():
license_kinds.update(kinds)
return license_kinds
else:
return self._license_kinds_map[module]
license_collector = collect_licenses.LicenseCollector(self._install_dir)
license_collector.run(license_text_path)
license_collector.run(module)
return license_collector.license_kinds
def _get_license_texts(self, module):
if self._license_in_json:
return {'{notice_dir}/{license_text}'.format(
notice_dir=utils.NOTICE_FILES_DIR_NAME,
license_text=license_text)
for license_text in self._license_texts_map[module]}
else:
return {'{notice_dir}/{module}.txt'.format(
notice_dir=utils.NOTICE_FILES_DIR_NAME,
module=module)}
def _gen_license(self):
""" Generates license module.
@@ -333,7 +336,7 @@ class GenBuildFile(object):
ind=self.INDENT,
version=self._vndk_version,
license_kinds=license_kinds_string,
notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '*.txt')))
notice_files=os.path.join(utils.NOTICE_FILES_DIR_PATH, '**', '*')))
def _get_versioned_name(self,
prebuilt,
@@ -451,29 +454,43 @@ class GenBuildFile(object):
def _gen_notice_license(self, module):
"""Generates a notice license build rule for a given module.
When genererating each notice license, collect
modules_with_restricted_lic, the list of modules that are under the GPL.
Args:
notice: string, module name
module: string, module name
"""
license_kinds = self._get_license_kinds('{notice_dir}/{module}.txt'.format(
notice_dir=utils.NOTICE_FILES_DIR_NAME,
module=module))
def has_restricted_license(license_kinds):
for lic in license_kinds:
if 'GPL' in lic:
return True
return False
license_kinds = self._get_license_kinds(module)
if has_restricted_license(license_kinds):
self.modules_with_restricted_lic.add(module)
license_kinds_string = ''
for license_kind in sorted(license_kinds):
license_kinds_string += '{ind}{ind}"{license_kind}",\n'.format(
ind=self.INDENT, license_kind=license_kind)
license_texts = self._get_license_texts(module)
license_texts_string = ''
for license_text in sorted(license_texts):
license_texts_string += '{ind}{ind}"{license_text}",\n'.format(
ind=self.INDENT, license_text=license_text)
return ('license {{\n'
'{ind}name: "{license_name}",\n'
'{ind}license_kinds: [\n'
'{license_kinds}'
'{ind}],\n'
'{ind}license_text: ["{notice_dir}/{module}.txt"],\n'
'{ind}license_text: [\n'
'{license_texts}'
'{ind}],\n'
'}}\n'.format(
ind=self.INDENT,
license_name=self._get_notice_license_name(module),
license_kinds=license_kinds_string,
module=module,
notice_dir=utils.NOTICE_FILES_DIR_NAME))
license_texts=license_texts_string))
def _get_notice_license_name(self, module):
""" Gets a notice license module name for a given module.
@@ -551,6 +568,18 @@ class GenBuildFile(object):
return True
return False
def get_license_prop(name):
"""Returns the license prop build rule.
Args:
name: string, name of the module
"""
if name in self._license_kinds_map:
return '{ind}licenses: ["{license}"],\n'.format(
ind=self.INDENT,
license=self._get_notice_license_name(name))
return ''
def get_notice_file(prebuilts):
"""Returns build rule for notice file (attribute 'licenses').
@@ -607,7 +636,7 @@ class GenBuildFile(object):
name=name))
def rename_generated_dirs(dirs):
# Reame out/soong/.intermedaites to generated-headers for better readability.
# Rename out/soong/.intermediates to generated-headers for better readability.
return [d.replace(utils.SOONG_INTERMEDIATES_DIR, utils.GENERATED_HEADERS_DIR, 1) for d in dirs]
for src in sorted(src_paths):
@@ -640,6 +669,10 @@ class GenBuildFile(object):
'relative_install_path: "{path}",\n').format(
ind=self.INDENT,
path=props['RelativeInstallPath'])
if 'LicenseKinds' in props:
self._license_kinds_map[name].update(props['LicenseKinds'])
if 'LicenseTexts' in props:
self._license_texts_map[name].update(props['LicenseTexts'])
arch_props += ('{ind}{ind}{arch}: {{\n'
'{include_dirs}'
@@ -700,8 +733,11 @@ class GenBuildFile(object):
vndk_sp=vndk_sp,
vndk_private=vndk_private))
notice = get_notice_file(srcs)
arch_props = get_arch_props(name, arch, src_paths)
if self._license_in_json:
license = get_license_prop(name)
else:
license = get_notice_file(srcs)
binder32bit = ''
if is_binder32:
@@ -715,7 +751,7 @@ class GenBuildFile(object):
'{ind}vendor_available: true,\n'
'{product_available}'
'{vndk_props}'
'{notice}'
'{license}'
'{arch_props}'
'}}\n'.format(
ind=self.INDENT,
@@ -725,7 +761,7 @@ class GenBuildFile(object):
binder32bit=binder32bit,
product_available=product_available,
vndk_props=vndk_props,
notice=notice,
license=license,
arch_props=arch_props))
@@ -765,9 +801,11 @@ def main():
utils.set_logging_config(args.verbose)
buildfile_generator = GenBuildFile(install_dir, vndk_version)
# To parse json information, read and generate arch android.bp using
# generate_android_bp() first.
buildfile_generator.generate_android_bp()
buildfile_generator.generate_root_android_bp()
buildfile_generator.generate_common_android_bp()
buildfile_generator.generate_android_bp()
logging.info('Done.')

21
vndk/snapshot/update.py
View File

@@ -114,11 +114,13 @@ def gather_notice_files(install_dir):
notices_dir_per_arch = os.path.join(arch, utils.NOTICE_FILES_DIR_NAME)
if os.path.isdir(notices_dir_per_arch):
for notice_file in glob.glob(
'{}/*.txt'.format(notices_dir_per_arch)):
if not os.path.isfile(
os.path.join(common_notices_dir,
os.path.basename(notice_file))):
shutil.copy(notice_file, common_notices_dir)
'{}/**'.format(notices_dir_per_arch), recursive=True):
if os.path.isfile(notice_file):
rel_path = os.path.relpath(notice_file, notices_dir_per_arch)
target_path = os.path.join(common_notices_dir, rel_path)
if not os.path.isfile(target_path):
os.makedirs(os.path.dirname(target_path), exist_ok=True)
shutil.copy(notice_file, target_path)
shutil.rmtree(notices_dir_per_arch)
@@ -156,15 +158,17 @@ def post_processe_files_if_needed(vndk_version):
def update_buildfiles(buildfile_generator):
# To parse json information, read and generate arch android.bp using
# generate_android_bp() first.
logging.info('Generating Android.bp files...')
buildfile_generator.generate_android_bp()
logging.info('Generating root Android.bp file...')
buildfile_generator.generate_root_android_bp()
logging.info('Generating common/Android.bp file...')
buildfile_generator.generate_common_android_bp()
logging.info('Generating Android.bp files...')
buildfile_generator.generate_android_bp()
def copy_owners(root_dir, install_dir):
path = os.path.dirname(__file__)
shutil.copy(os.path.join(root_dir, path, 'OWNERS'), install_dir)
@@ -257,6 +261,7 @@ def run(vndk_version, branch, build_id, local, use_current_branch, remote,
if not local_path and not branch.startswith('android'):
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
buildfile_generator.modules_with_restricted_lic,
temp_artifact_dir, remote)
check_gpl_license(license_checker)
logging.info(