From a2504712decce9246c826923e0fcaa2b585ab349 Mon Sep 17 00:00:00 2001 From: Jae Shin Date: Wed, 18 Apr 2018 18:00:26 +0900 Subject: [PATCH] Prevent manifest file from being added to prebuilts Change logic that handles the manifest file so that the manifest is not added to the prebuilts/ directory but remains in a temporary dir for GPL checking. At the end of the update.py script, the manifest file is deleted when the temporary dir is deleted. Also modify how check_gpl_license.py is run independently since the manifest file has to be fetched first. Test: python update.py -b oc-mr1-treble-dev --build 4728343 -vv 27 Bug: 77546642 Change-Id: I7749e2998cd12325b9c6275da5e697e328137b73 --- vndk/snapshot/check_gpl_license.py | 35 +++++++--- vndk/snapshot/gen_buildfiles.py | 15 ++-- vndk/snapshot/update.py | 108 ++++++++++++++--------------- vndk/snapshot/utils.py | 20 +++++- 4 files changed, 107 insertions(+), 71 deletions(-) diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py index aa4535061..25004440c 100644 --- a/vndk/snapshot/check_gpl_license.py +++ b/vndk/snapshot/check_gpl_license.py @@ -18,7 +18,9 @@ import argparse import glob import os +import shutil import subprocess +import tempfile import xml.etree.ElementTree as xml_tree import utils @@ -33,7 +35,7 @@ class GPLChecker(object): MANIFEST_XML = utils.MANIFEST_FILE_NAME MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME - def __init__(self, install_dir, android_build_top): + def __init__(self, install_dir, android_build_top, temp_artifact_dir): """GPLChecker constructor. Args: @@ -43,14 +45,16 @@ class GPLChecker(object): """ self._android_build_top = android_build_top self._install_dir = install_dir - self._manifest_file = os.path.join(install_dir, - utils.MANIFEST_FILE_PATH) + self._manifest_file = os.path.join(temp_artifact_dir, + self.MANIFEST_XML) self._notice_files_dir = os.path.join(install_dir, utils.NOTICE_FILES_DIR_PATH) if not os.path.isfile(self._manifest_file): - raise RuntimeError('{manifest} not found at {manifest_file}'.format( - manifest=self.MANIFEST_XML, manifest_file=self._manifest_file)) + raise RuntimeError( + '{manifest} not found at {manifest_file}'.format( + manifest=self.MANIFEST_XML, + manifest_file=self._manifest_file)) def _parse_module_paths(self): """Parses the module_path.txt files into a dictionary, @@ -115,8 +119,8 @@ class GPLChecker(object): notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) if len(notice_files) == 0: - raise RuntimeError( - 'No license files found in {}'.format(self._notice_files_dir)) + raise RuntimeError('No license files found in {}'.format( + self._notice_files_dir)) gpl_projects = [] pattern = 'GENERAL PUBLIC LICENSE' @@ -173,6 +177,8 @@ def get_args(): parser.add_argument( 'vndk_version', type=int, help='VNDK snapshot version to check, e.g. "27".') + parser.add_argument('-b', '--branch', help='Branch to pull manifest from.') + parser.add_argument('--build', help='Build number to pull manifest from.') return parser.parse_args() @@ -194,12 +200,25 @@ def main(): 'Please provide valid VNDK version. {} does not exist.' .format(install_dir)) - license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) + temp_artifact_dir = tempfile.mkdtemp() + os.chdir(temp_artifact_dir) + manifest_pattern = 'manifest_{}.xml'.format(args.build) + print 'Fetching {file} from {branch} (bid: {build})'.format( + file=manifest_pattern, branch=args.branch, build=args.build) + manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) + utils.fetch_artifact(args.branch, args.build, manifest_pattern, + manifest_dest) + + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + temp_artifact_dir) try: license_checker.check_gpl_projects() except ValueError as error: print error raise + finally: + print 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir) + shutil.rmtree(temp_artifact_dir) if __name__ == '__main__': diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index cb85a449f..e103dd56b 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -45,7 +45,6 @@ class GenBuildFile(object): ... (other {SNAPSHOT_VARIANT}/ directories) common/ Android.mk - manifest.xml NOTICE_FILES/ (license files, e.g. libfoo.so.txt) """ @@ -297,13 +296,13 @@ class GenBuildFile(object): variant_path = os.path.join(self._install_dir, variant) src_paths = utils.find(variant_path, [prebuilt]) for src in sorted(src_paths): - arch_srcs += ( - '{ind}{ind}{arch}: {{\n' - '{ind}{ind}{ind}srcs: ["{src}"],\n' - '{ind}{ind}}},\n'.format( - ind=self.INDENT, - arch=utils.arch_from_path(os.path.join(variant, src)), - src=src)) + arch_srcs += ('{ind}{ind}{arch}: {{\n' + '{ind}{ind}{ind}srcs: ["{src}"],\n' + '{ind}{ind}}},\n'.format( + ind=self.INDENT, + arch=utils.arch_from_path( + os.path.join(variant, src)), + src=src)) arch_srcs += '{ind}}},\n'.format(ind=self.INDENT) return arch_srcs diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index 4209e3302..c74410f82 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -46,15 +46,6 @@ def check_call(cmd): subprocess.check_call(cmd) -def fetch_artifact(branch, build, pattern, destination='.'): - fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' - cmd = [ - fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid', - build, pattern, destination - ] - check_call(cmd) - - def start_branch(build): branch_name = 'update-' + (build or 'local') logger().info('Creating branch {branch} in {dir}'.format( @@ -75,7 +66,7 @@ def remove_old_snapshot(install_dir): sys.exit(1) -def install_snapshot(branch, build, install_dir): +def install_snapshot(branch, build, install_dir, temp_artifact_dir): """Installs VNDK snapshot build artifacts to prebuilts/vndk/v{version}. 1) Fetch build artifacts from Android Build server or from local DIST_DIR @@ -85,48 +76,38 @@ def install_snapshot(branch, build, install_dir): branch: string or None, branch name of build artifacts build: string or None, build number of build artifacts install_dir: string, directory to install VNDK snapshot + temp_artifact_dir: string, temp directory to hold build artifacts fetched + from Android Build server. For 'local' option, is set to None. """ artifact_pattern = 'android-vndk-*.zip' - try: - if branch and build: - tempdir = tempfile.mkdtemp() - artifact_dir = tempdir + if branch and build: + artifact_dir = temp_artifact_dir + os.chdir(temp_artifact_dir) + logger().info('Fetching {pattern} from {branch} (bid: {build})'.format( + pattern=artifact_pattern, branch=branch, build=build)) + utils.fetch_artifact(branch, build, artifact_pattern) - os.chdir(tempdir) - logger().info( - 'Fetching {pattern} from {branch} (bid: {build})' - .format(pattern=artifact_pattern, branch=branch, build=build)) - fetch_artifact(branch, build, artifact_pattern) + manifest_pattern = 'manifest_{}.xml'.format(build) + logger().info('Fetching {file} from {branch} (bid: {build})'.format( + file=manifest_pattern, branch=branch, build=build)) + utils.fetch_artifact(branch, build, manifest_pattern, + utils.MANIFEST_FILE_NAME) - manifest_pattern = 'manifest_{}.xml'.format(build) - manifest_name = utils.MANIFEST_FILE_NAME - logger().info( - 'Fetching {file} from {branch} (bid: {build})'.format( - file=manifest_pattern, branch=branch, build=build)) - fetch_artifact(branch, build, manifest_pattern, manifest_name) - shutil.move(manifest_name, - os.path.join(install_dir, utils.COMMON_DIR_PATH)) + os.chdir(install_dir) + else: + logger().info('Fetching local VNDK snapshot from {}'.format(DIST_DIR)) + artifact_dir = DIST_DIR - os.chdir(install_dir) - else: - logger().info( - 'Fetching local VNDK snapshot from {}'.format(DIST_DIR)) - artifact_dir = DIST_DIR - - artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) - artifact_cnt = len(artifacts) - if artifact_cnt < 4: - raise RuntimeError( - 'Expected four android-vndk-*.zip files in {path}. Instead ' - 'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt)) - for artifact in artifacts: - logger().info('Unzipping VNDK snapshot: {}'.format(artifact)) - check_call(['unzip', '-q', artifact, '-d', install_dir]) - finally: - if branch and build: - logger().info('Deleting tempdir: {}'.format(tempdir)) - shutil.rmtree(tempdir) + artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) + artifact_cnt = len(artifacts) + if artifact_cnt < 4: + raise RuntimeError( + 'Expected four android-vndk-*.zip files in {path}. Instead ' + 'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt)) + for artifact in artifacts: + logger().info('Unzipping VNDK snapshot: {}'.format(artifact)) + check_call(['unzip', '-q', artifact, '-d', install_dir]) def gather_notice_files(install_dir): @@ -255,16 +236,35 @@ def main(): remove_old_snapshot(install_dir) os.makedirs(utils.COMMON_DIR_PATH) - install_snapshot(args.branch, args.build, install_dir) - gather_notice_files(install_dir) - revise_ld_config_txt_if_needed(vndk_version) - - buildfile_generator = GenBuildFile(install_dir, vndk_version) - update_buildfiles(buildfile_generator) + temp_artifact_dir = None if not args.local: - license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) - check_gpl_license(license_checker) + temp_artifact_dir = tempfile.mkdtemp() + + install_status = True + try: + install_snapshot(args.branch, args.build, install_dir, + temp_artifact_dir) + gather_notice_files(install_dir) + revise_ld_config_txt_if_needed(vndk_version) + + buildfile_generator = GenBuildFile(install_dir, vndk_version) + update_buildfiles(buildfile_generator) + + if not args.local: + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, + temp_artifact_dir) + check_gpl_license(license_checker) + except: + logger().info('FAILED TO INSTALL SNAPSHOT') + install_status = False + finally: + if temp_artifact_dir: + logger().info( + 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) + shutil.rmtree(temp_artifact_dir) + + if not args.local and install_status: commit(args.branch, args.build, vndk_version) diff --git a/vndk/snapshot/utils.py b/vndk/snapshot/utils.py index b8fff0ae0..9190778a1 100644 --- a/vndk/snapshot/utils.py +++ b/vndk/snapshot/utils.py @@ -19,6 +19,7 @@ import glob import os import re +import subprocess import sys # Global Keys @@ -28,7 +29,6 @@ COMMON_DIR_PATH = COMMON_DIR_NAME ANDROID_MK_PATH = os.path.join(COMMON_DIR_PATH, 'Android.mk') CONFIG_DIR_PATH_PATTERN = '*/configs' MANIFEST_FILE_NAME = 'manifest.xml' -MANIFEST_FILE_PATH = os.path.join(COMMON_DIR_PATH, MANIFEST_FILE_NAME) MODULE_PATHS_FILE_NAME = 'module_paths.txt' NOTICE_FILES_DIR_NAME = 'NOTICE_FILES' NOTICE_FILES_DIR_PATH = os.path.join(COMMON_DIR_PATH, NOTICE_FILES_DIR_NAME) @@ -113,3 +113,21 @@ def find(path, names): rel_to_root = abspath.replace(os.path.abspath(path), '') found.append(rel_to_root[1:]) # strip leading / return found + + +def fetch_artifact(branch, build, pattern, destination='.'): + """Fetches build artifacts from Android Build server. + + Args: + branch: string, branch to pull build artifacts from + build: string, build number to pull build artifacts from + pattern: string, pattern of build artifact file name + destination: string, destination to pull build artifact to + """ + fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' + cmd = [ + fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid', + build, pattern, destination + ] + print 'Running `{}`'.format(' '.join(cmd)) + subprocess.check_call(cmd)