From 5233fe115b635d2e19ea7c0ee9b93201a482b498 Mon Sep 17 00:00:00 2001 From: Jae Shin Date: Mon, 18 Dec 2017 22:29:48 +0900 Subject: [PATCH] Add script to check GPL projects have released SHA Checks that the source tree to which the VNDK snapshot is being installed has the sources for all GPL prebuilts by confirming the git projects have the SHA values recorded in manifest.xml. Bug: 70603439 Test: python update.py [options] Test: python check_gpl_license.py [options] Change-Id: I5b34271cd0dab737187211f52b2fdce6dbd94e2e --- vndk/snapshot/check_gpl_license.py | 204 +++++++++++++++++++++++++++++ vndk/snapshot/gen_buildfiles.py | 53 ++------ vndk/snapshot/update.py | 53 ++++---- vndk/snapshot/utils.py | 75 +++++++++++ 4 files changed, 317 insertions(+), 68 deletions(-) create mode 100644 vndk/snapshot/check_gpl_license.py create mode 100644 vndk/snapshot/utils.py diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py new file mode 100644 index 000000000..228888e52 --- /dev/null +++ b/vndk/snapshot/check_gpl_license.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import glob +import os +import subprocess +import xml.etree.ElementTree as xml_tree + +import utils + + +class GPLChecker(object): + """Checks that all GPL projects in a VNDK snapshot have released sources. + + Makes sure that the current source tree have the sources for all GPL + prebuilt libraries in a specified VNDK snapshot version. + """ + MANIFEST_XML = 'manifest.xml' + MODULE_PATHS_TXT = 'module_paths.txt' + + def __init__(self, install_dir, android_build_top): + """GPLChecker constructor. + + Args: + install_dir: string, absolute path to the prebuilts/vndk/v{version} + directory where the build files will be generated. + android_build_top: string, absolute path to ANDROID_BUILD_TOP + """ + self._android_build_top = android_build_top + self._install_dir = install_dir + self._manifest_file = os.path.join(install_dir, self.MANIFEST_XML) + self._notice_files_dir = os.path.join(install_dir, 'NOTICE_FILES') + + if not os.path.isfile(self._manifest_file): + raise RuntimeError('{manifest} not found in {install_dir}'.format( + manifest=self.MANIFEST_XML, install_dir=install_dir)) + + def _parse_module_paths(self): + """Parses the module_path.txt files into a dictionary, + + Returns: + module_paths: dict, e.g. {libfoo.so: some/path/here} + """ + module_paths = dict() + for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]): + file_path = os.path.join(self._install_dir, file) + with open(file_path, 'r') as f: + for line in f.read().strip().split('\n'): + paths = line.split(' ') + if len(paths) > 1: + if paths[0] not in module_paths: + module_paths[paths[0]] = paths[1] + return module_paths + + def _parse_manifest(self): + """Parses manifest.xml file and returns list of 'project' tags.""" + + root = xml_tree.parse(self._manifest_file).getroot() + return root.findall('project') + + def _get_revision(self, module_path, manifest_projects): + """Returns revision value recorded in manifest.xml for given project. + + Args: + module_path: string, project path relative to ANDROID_BUILD_TOP + manifest_projects: list of xml_tree.Element, list of 'project' tags + """ + revision = None + for project in manifest_projects: + path = project.get('path') + if module_path.startswith(path): + revision = project.get('revision') + break + return revision + + def _check_revision_exists(self, revision, git_project_path): + """Checks whether a revision is found in a git project of current tree. + + Args: + revision: string, revision value recorded in manifest.xml + git_project_path: string, path relative to ANDROID_BUILD_TOP + """ + path = utils.join_realpath(self._android_build_top, git_project_path) + try: + subprocess.check_call( + ['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)]) + return True + except subprocess.CalledProcessError: + return False + + def check_gpl_projects(self): + """Checks that all GPL projects have released sources. + + Raises: + ValueError: There are GPL projects with unreleased sources. + """ + print 'Starting license check for GPL projects...' + + notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) + if len(notice_files) == 0: + raise RuntimeError( + 'No license files found in {}'.format(self._notice_files_dir)) + + gpl_projects = [] + pattern = 'GENERAL PUBLIC LICENSE' + for notice_file_path in notice_files: + with open(notice_file_path, 'r') as notice_file: + if pattern in notice_file.read(): + lib_name = os.path.splitext( + os.path.basename(notice_file_path))[0] + gpl_projects.append(lib_name) + + if not gpl_projects: + print 'No GPL projects found.' + return + + print 'GPL projects found:', ', '.join(gpl_projects) + + module_paths = self._parse_module_paths() + manifest_projects = self._parse_manifest() + released_projects = [] + unreleased_projects = [] + + for lib in gpl_projects: + if lib in module_paths: + module_path = module_paths[lib] + revision = self._get_revision(module_path, manifest_projects) + if not revision: + raise RuntimeError( + 'No project found for {path} in {manifest}'.format( + path=module_path, manifest=self.MANIFEST_XML)) + revision_exists = self._check_revision_exists( + revision, module_path) + if not revision_exists: + unreleased_projects.append((lib, module_path)) + else: + released_projects.append((lib, module_path)) + else: + raise RuntimeError( + 'No module path was found for {lib} in {module_paths}'. + format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) + + if released_projects: + print 'Released GPL projects:', released_projects + + if unreleased_projects: + raise ValueError( + ('FAIL: The following GPL projects have NOT been released in ' + 'current tree: {}'.format(unreleased_projects))) + + print 'PASS: All GPL projects have source in current tree.' + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'vndk_version', type=int, + help='VNDK snapshot version to check, e.g. "27".') + return parser.parse_args() + + +def main(): + """For local testing purposes. + + Note: VNDK snapshot must be already installed under + prebuilts/vndk/v{version}. + """ + ANDROID_BUILD_TOP = utils.get_android_build_top() + PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, + 'prebuilts/vndk') + + args = get_args() + vndk_version = args.vndk_version + install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) + if not os.path.isdir(install_dir): + raise ValueError( + 'Please provide valid VNDK version. {} does not exist.' + .format(install_dir)) + + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) + try: + license_checker.check_gpl_projects() + except ValueError as error: + print error + raise + + +if __name__ == '__main__': + main() diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index 23bd97682..ba238ba8b 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -18,6 +18,8 @@ import os import sys +import utils + class GenBuildFile(object): """Generates Android.mk and Android.bp for prebuilts/vndk/v{version}.""" @@ -53,7 +55,7 @@ class GenBuildFile(object): txt_filename: string, file name in VNDK snapshot """ prebuilt_list = [] - txts = find(self._install_dir, [txt_filename]) + txts = utils.find(self._install_dir, [txt_filename]) for txt in txts: path_to_txt = os.path.join(self._install_dir, txt) with open(path_to_txt, 'r') as f: @@ -126,7 +128,7 @@ class GenBuildFile(object): Args: prebuilt: string, name of ETC prebuilt object """ - etc_path = find(self._install_dir, prebuilt)[0] + etc_path = utils.find(self._install_dir, prebuilt)[0] etc_sub_path = etc_path[etc_path.index('/') + 1:] return ( @@ -202,7 +204,7 @@ class GenBuildFile(object): notice = '' notice_file_name = '{}.txt'.format(prebuilt) notices_dir = os.path.join(self._install_dir, 'NOTICE_FILES') - notice_files = find(notices_dir, [notice_file_name]) + notice_files = utils.find(notices_dir, [notice_file_name]) if len(notice_files) > 0: notice = '{ind}notice: "{notice_file_path}",\n'.format( ind=self.INDENT, @@ -241,15 +243,13 @@ class GenBuildFile(object): prebuilt: string, name of prebuilt object """ arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT) - src_paths = find(self._install_dir, [prebuilt]) - # if len(src_paths) < 4: - # print prebuilt, src_paths + src_paths = utils.find(self._install_dir, [prebuilt]) for src in sorted(src_paths): arch_srcs += ('{ind}{ind}{arch}: {{\n' '{ind}{ind}{ind}srcs: ["{src}"],\n' '{ind}{ind}}},\n'.format( ind=self.INDENT, - arch=arch_from_path(src), + arch=utils.arch_from_path(src), src=src)) arch_srcs += '{ind}}},\n'.format(ind=self.INDENT) return arch_srcs @@ -287,48 +287,15 @@ class GenBuildFile(object): arch_srcs=arch_srcs)) -def find(path, names): - """Finds a list of files in a directory that match the given names. - - Args: - path: string, absolute path of directory from which to find files - names: list of strings, names of the files to find - """ - found = [] - for root, _, files in os.walk(path): - for file_name in sorted(files): - if file_name in names: - abspath = os.path.abspath(os.path.join(root, file_name)) - rel_to_root = abspath.replace(os.path.abspath(path), '') - found.append(rel_to_root[1:]) # strip leading / - return found - - -def arch_from_path(path): - """Extracts archfrom given VNDK snapshot path. - - Args: - path: string, path relative to prebuilts/vndk/v{version} - - Returns: - arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64") - """ - return path.split('/')[0].split('-')[1] - - def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ - ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') - if not ANDROID_BUILD_TOP: - print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' - '\'. build/envsetup.sh; lunch \'. Exiting script.') - sys.exit(1) - PREBUILTS_VNDK_DIR = os.path.realpath( - os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk')) + ANDROID_BUILD_TOP = utils.get_android_build_top() + PREBUILTS_VNDK_DIR = utils.join_realpath( + ANDROID_BUILD_TOP, 'prebuilts/vndk') vndk_version = 27 # set appropriately install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index e10a5760c..1038a9985 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -26,24 +26,14 @@ import sys import tempfile import textwrap +import utils + +from check_gpl_license import GPLChecker from gen_buildfiles import GenBuildFile -ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') -if not ANDROID_BUILD_TOP: - print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' - '\'. build/envsetup.sh; lunch \'. Exiting script.') - sys.exit(1) - -DIST_DIR = os.getenv('DIST_DIR') -if not DIST_DIR: - OUT_DIR = os.getenv('OUT_DIR') - if OUT_DIR: - DIST_DIR = os.path.realpath(os.path.join(OUT_DIR, 'dist')) - else: - DIST_DIR = os.path.realpath(os.path.join(ANDROID_BUILD_TOP, 'out/dist')) - -PREBUILTS_VNDK_DIR = os.path.realpath( - os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk')) +ANDROID_BUILD_TOP = utils.get_android_build_top() +DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP)) +PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') def logger(): @@ -103,21 +93,23 @@ def install_snapshot(branch, build, install_dir): artifact_dir = tempdir os.chdir(tempdir) - logger().info('Fetching {pattern} from {branch} (bid: {build})' + logger().info( + 'Fetching {pattern} from {branch} (bid: {build})' .format(pattern=artifact_pattern, branch=branch, build=build)) fetch_artifact(branch, build, artifact_pattern) manifest_pattern = 'manifest_{}.xml'.format(build) manifest_name = 'manifest.xml' - logger().info('Fetching {file} from {branch} (bid: {build})'.format( - file=manifest_pattern, branch=branch, build=build)) + logger().info( + 'Fetching {file} from {branch} (bid: {build})'.format( + file=manifest_pattern, branch=branch, build=build)) fetch_artifact(branch, build, manifest_pattern, manifest_name) shutil.move(manifest_name, install_dir) os.chdir(install_dir) else: - logger().info('Fetching local VNDK snapshot from {}'.format( - DIST_DIR)) + logger().info( + 'Fetching local VNDK snapshot from {}'.format(DIST_DIR)) artifact_dir = DIST_DIR artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) @@ -148,9 +140,10 @@ def gather_notice_files(): notices_dir_per_arch = os.path.join(arch_dir, notices_dir_name) if os.path.isdir(notices_dir_per_arch): for notice_file in glob.glob( - '{}/*.txt'.format(notices_dir_per_arch)): - if not os.path.isfile(os.path.join(notices_dir_name, - os.path.basename(notice_file))): + '{}/*.txt'.format(notices_dir_per_arch)): + if not os.path.isfile( + os.path.join(notices_dir_name, + os.path.basename(notice_file))): shutil.copy(notice_file, notices_dir_name) shutil.rmtree(notices_dir_per_arch) @@ -163,6 +156,14 @@ def update_buildfiles(buildfile_generator): buildfile_generator.generate_android_bp() +def check_gpl_license(license_checker): + try: + license_checker.check_gpl_projects() + except ValueError as error: + print '***CANNOT INSTALL VNDK SNAPSHOT***', error + raise + + def commit(branch, build, version): logger().info('Making commit...') check_call(['git', 'add', '.']) @@ -186,7 +187,7 @@ def get_args(): help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of ' 'Android Build server.')) parser.add_argument( - '--use-current-branch',action='store_true', + '--use-current-branch', action='store_true', help='Perform the update in the current branch. Do not repo start.') parser.add_argument( '-v', '--verbose', action='count', default=0, @@ -239,6 +240,8 @@ def main(): update_buildfiles(buildfile_generator) if not args.local: + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) + check_gpl_license(license_checker) commit(args.branch, args.build, vndk_version) diff --git a/vndk/snapshot/utils.py b/vndk/snapshot/utils.py new file mode 100644 index 000000000..b21225b26 --- /dev/null +++ b/vndk/snapshot/utils.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utility functions for VNDK snapshot.""" + +import os +import sys + + +def get_android_build_top(): + ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') + if not ANDROID_BUILD_TOP: + print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' + '\'. build/envsetup.sh; lunch \'. Exiting script.') + sys.exit(1) + return ANDROID_BUILD_TOP + + +def join_realpath(root, *args): + return os.path.realpath(os.path.join(root, *args)) + + +def _get_dir_from_env(env_var, default): + return os.path.realpath(os.getenv(env_var, default)) + + +def get_out_dir(android_build_top): + return _get_dir_from_env('OUT_DIR', join_realpath(android_build_top, + 'out')) + + +def get_dist_dir(out_dir): + return _get_dir_from_env('DIST_DIR', join_realpath(out_dir, 'dist')) + + +def arch_from_path(path): + """Extracts arch from given VNDK snapshot path. + + Args: + path: string, path relative to prebuilts/vndk/v{version} + + Returns: + arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64") + """ + return path.split('/')[0].split('-')[1] + + +def find(path, names): + """Returns a list of files in a directory that match the given names. + + Args: + path: string, absolute path of directory from which to find files + names: list of strings, names of the files to find + """ + found = [] + for root, _, files in os.walk(path): + for file_name in sorted(files): + if file_name in names: + abspath = os.path.abspath(os.path.join(root, file_name)) + rel_to_root = abspath.replace(os.path.abspath(path), '') + found.append(rel_to_root[1:]) # strip leading / + return found