diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py index 25004440c..bb7f3e80b 100644 --- a/vndk/snapshot/check_gpl_license.py +++ b/vndk/snapshot/check_gpl_license.py @@ -17,6 +17,7 @@ import argparse import glob +import logging import os import shutil import subprocess @@ -25,6 +26,8 @@ import xml.etree.ElementTree as xml_tree import utils +logger = utils.logger(__name__) + class GPLChecker(object): """Checks that all GPL projects in a VNDK snapshot have released sources. @@ -57,7 +60,7 @@ class GPLChecker(object): manifest_file=self._manifest_file)) def _parse_module_paths(self): - """Parses the module_path.txt files into a dictionary, + """Parses the module_paths.txt files into a dictionary, Returns: module_paths: dict, e.g. {libfoo.so: some/path/here} @@ -103,10 +106,11 @@ class GPLChecker(object): """ path = utils.join_realpath(self._android_build_top, git_project_path) try: - subprocess.check_call( - ['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)]) + cmd = ['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)] + utils.check_call(cmd, logger) return True - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as error: + logger.error('Error: {}'.format(error)) return False def check_gpl_projects(self): @@ -115,7 +119,7 @@ class GPLChecker(object): Raises: ValueError: There are GPL projects with unreleased sources. """ - print 'Starting license check for GPL projects...' + logger.info('Starting license check for GPL projects...') notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) if len(notice_files) == 0: @@ -132,10 +136,10 @@ class GPLChecker(object): gpl_projects.append(lib_name) if not gpl_projects: - print 'No GPL projects found.' + logger.info('No GPL projects found.') return - print 'GPL projects found:', ', '.join(gpl_projects) + logger.info('GPL projects found: {}'.format(', '.join(gpl_projects))) module_paths = self._parse_module_paths() manifest_projects = self._parse_manifest() @@ -162,23 +166,30 @@ class GPLChecker(object): format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) if released_projects: - print 'Released GPL projects:', released_projects + logger.info('Released GPL projects: {}'.format(released_projects)) if unreleased_projects: raise ValueError( ('FAIL: The following GPL projects have NOT been released in ' 'current tree: {}'.format(unreleased_projects))) - print 'PASS: All GPL projects have source in current tree.' + logger.info('PASS: All GPL projects have source in current tree.') def get_args(): parser = argparse.ArgumentParser() parser.add_argument( - 'vndk_version', type=int, + 'vndk_version', + type=int, help='VNDK snapshot version to check, e.g. "27".') parser.add_argument('-b', '--branch', help='Branch to pull manifest from.') parser.add_argument('--build', help='Build number to pull manifest from.') + parser.add_argument( + '-v', + '--verbose', + action='count', + default=0, + help='Increase output verbosity, e.g. "-v", "-vv".') return parser.parse_args() @@ -199,13 +210,14 @@ def main(): raise ValueError( 'Please provide valid VNDK version. {} does not exist.' .format(install_dir)) + utils.set_logging_config(args.verbose) temp_artifact_dir = tempfile.mkdtemp() os.chdir(temp_artifact_dir) manifest_pattern = 'manifest_{}.xml'.format(args.build) - print 'Fetching {file} from {branch} (bid: {build})'.format( - file=manifest_pattern, branch=args.branch, build=args.build) manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) + logger.info('Fetching {file} from {branch} (bid: {build})'.format( + file=manifest_pattern, branch=args.branch, build=args.build)) utils.fetch_artifact(args.branch, args.build, manifest_pattern, manifest_dest) @@ -214,12 +226,14 @@ def main(): try: license_checker.check_gpl_projects() except ValueError as error: - print error + logger.error('Error: {}'.format(error)) raise finally: - print 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir) + logger.info('Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) + logger.info('Done.') + if __name__ == '__main__': main() diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index e103dd56b..c73941a61 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -15,12 +15,16 @@ # limitations under the License. # +import argparse import glob +import logging import os import sys import utils +logger = utils.logger(__name__) + class GenBuildFile(object): """Generates Android.mk and Android.bp for VNDK snapshot. @@ -342,6 +346,21 @@ class GenBuildFile(object): arch_srcs=arch_srcs)) +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'vndk_version', + type=int, + help='VNDK snapshot version to install, e.g. "27".') + parser.add_argument( + '-v', + '--verbose', + action='count', + default=0, + help='Increase output verbosity, e.g. "-v", "-vv".') + return parser.parse_args() + + def main(): """For local testing purposes. @@ -352,13 +371,21 @@ def main(): PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') - vndk_version = 27 # set appropriately + args = get_args() + vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) + if not os.path.isdir(install_dir): + raise ValueError( + 'Please provide valid VNDK version. {} does not exist.' + .format(install_dir)) + utils.set_logging_config(args.verbose) buildfile_generator = GenBuildFile(install_dir, vndk_version) buildfile_generator.generate_android_mk() buildfile_generator.generate_android_bp() + logger.info('Done.') + if __name__ == '__main__': main() diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index c74410f82..f22cc1f79 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -36,25 +36,18 @@ ANDROID_BUILD_TOP = utils.get_android_build_top() DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP)) PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') - -def logger(): - return logging.getLogger(__name__) - - -def check_call(cmd): - logger().debug('Running `{}`'.format(' '.join(cmd))) - subprocess.check_call(cmd) +logger = utils.logger(__name__) def start_branch(build): branch_name = 'update-' + (build or 'local') - logger().info('Creating branch {branch} in {dir}'.format( + logger.info('Creating branch {branch} in {dir}'.format( branch=branch_name, dir=os.getcwd())) - check_call(['repo', 'start', branch_name, '.']) + utils.check_call(['repo', 'start', branch_name, '.'], logger) def remove_old_snapshot(install_dir): - logger().info('Removing any old files in {}'.format(install_dir)) + logger.info('Removing any old files in {}'.format(install_dir)) for file in glob.glob('{}/*'.format(install_dir)): try: if os.path.isfile(file): @@ -62,7 +55,7 @@ def remove_old_snapshot(install_dir): elif os.path.isdir(file): shutil.rmtree(file) except Exception as error: - print error + logger.error('Error: {}'.format(error)) sys.exit(1) @@ -84,19 +77,19 @@ def install_snapshot(branch, build, install_dir, temp_artifact_dir): if branch and build: artifact_dir = temp_artifact_dir os.chdir(temp_artifact_dir) - logger().info('Fetching {pattern} from {branch} (bid: {build})'.format( + logger.info('Fetching {pattern} from {branch} (bid: {build})'.format( pattern=artifact_pattern, branch=branch, build=build)) utils.fetch_artifact(branch, build, artifact_pattern) manifest_pattern = 'manifest_{}.xml'.format(build) - logger().info('Fetching {file} from {branch} (bid: {build})'.format( + logger.info('Fetching {file} from {branch} (bid: {build})'.format( file=manifest_pattern, branch=branch, build=build)) utils.fetch_artifact(branch, build, manifest_pattern, utils.MANIFEST_FILE_NAME) os.chdir(install_dir) else: - logger().info('Fetching local VNDK snapshot from {}'.format(DIST_DIR)) + logger.info('Fetching local VNDK snapshot from {}'.format(DIST_DIR)) artifact_dir = DIST_DIR artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) @@ -106,15 +99,16 @@ def install_snapshot(branch, build, install_dir, temp_artifact_dir): 'Expected four android-vndk-*.zip files in {path}. Instead ' 'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt)) for artifact in artifacts: - logger().info('Unzipping VNDK snapshot: {}'.format(artifact)) - check_call(['unzip', '-q', artifact, '-d', install_dir]) + logger.info('Unzipping VNDK snapshot: {}'.format(artifact)) + utils.check_call(['unzip', '-q', artifact, '-d', install_dir], logger) def gather_notice_files(install_dir): """Gathers all NOTICE files to a common NOTICE_FILES directory.""" common_notices_dir = utils.NOTICE_FILES_DIR_PATH - logger().info('Creating {} directory...'.format(common_notices_dir)) + logger.info('Creating {} directory to gather all NOTICE files...'.format( + common_notices_dir)) os.makedirs(common_notices_dir) for variant in utils.get_snapshot_variants(install_dir): notices_dir_per_variant = os.path.join(variant, @@ -138,6 +132,7 @@ def revise_ld_config_txt_if_needed(vndk_version): Args: vndk_version: string, version of VNDK snapshot """ + logger.info('Revising ld.config.txt for O-MR1...') if vndk_version == '27': re_pattern = '(system\/\${LIB}\/vndk(?:-sp)?)([:/]|$)' VNDK_INSTALL_DIR_RE = re.compile(re_pattern, flags=re.MULTILINE) @@ -151,10 +146,10 @@ def revise_ld_config_txt_if_needed(vndk_version): def update_buildfiles(buildfile_generator): - logger().info('Generating Android.mk file...') + logger.info('Generating Android.mk file...') buildfile_generator.generate_android_mk() - logger().info('Generating Android.bp files...') + logger.info('Generating Android.bp files...') buildfile_generator.generate_android_bp() @@ -162,37 +157,43 @@ def check_gpl_license(license_checker): try: license_checker.check_gpl_projects() except ValueError as error: - print '***CANNOT INSTALL VNDK SNAPSHOT***', error + logger.error('***CANNOT INSTALL VNDK SNAPSHOT***: {}'.format(error)) raise def commit(branch, build, version): - logger().info('Making commit...') - check_call(['git', 'add', '.']) + logger.info('Making commit...') + utils.check_call(['git', 'add', '.'], logger) message = textwrap.dedent("""\ Update VNDK snapshot v{version} to build {build}. Taken from branch {branch}.""").format( version=version, branch=branch, build=build) - check_call(['git', 'commit', '-m', message]) + utils.check_call(['git', 'commit', '-m', message], logger) def get_args(): parser = argparse.ArgumentParser() parser.add_argument( - 'vndk_version', type=int, + 'vndk_version', + type=int, help='VNDK snapshot version to install, e.g. "27".') parser.add_argument('-b', '--branch', help='Branch to pull build from.') parser.add_argument('--build', help='Build number to pull.') parser.add_argument( - '--local', action='store_true', + '--local', + action='store_true', help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of ' 'Android Build server.')) parser.add_argument( - '--use-current-branch', action='store_true', + '--use-current-branch', + action='store_true', help='Perform the update in the current branch. Do not repo start.') parser.add_argument( - '-v', '--verbose', action='count', default=0, + '-v', + '--verbose', + action='count', + default=0, help='Increase output verbosity, e.g. "-v", "-vv".') return parser.parse_args() @@ -225,9 +226,7 @@ def main(): 'Please request a new git project for prebuilts/vndk/v{ver} ' 'before installing new snapshot.'.format(ver=vndk_version)) - verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) - verbosity = min(args.verbose, 2) - logging.basicConfig(level=verbose_map[verbosity]) + utils.set_logging_config(args.verbose) os.chdir(install_dir) @@ -241,7 +240,6 @@ def main(): if not args.local: temp_artifact_dir = tempfile.mkdtemp() - install_status = True try: install_snapshot(args.branch, args.build, install_dir, temp_artifact_dir) @@ -255,17 +253,23 @@ def main(): license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, temp_artifact_dir) check_gpl_license(license_checker) - except: - logger().info('FAILED TO INSTALL SNAPSHOT') - install_status = False + logger.info( + 'Successfully updated VNDK snapshot v{}'.format(vndk_version)) + except Exception as error: + logger.error('FAILED TO INSTALL SNAPSHOT: {}'.format(error)) + raise finally: if temp_artifact_dir: - logger().info( + logger.info( 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) - if not args.local and install_status: + if not args.local: commit(args.branch, args.build, vndk_version) + logger.info('Successfully created commit for VNDK snapshot v{}'.format( + vndk_version)) + + logger.info('Done.') if __name__ == '__main__': diff --git a/vndk/snapshot/utils.py b/vndk/snapshot/utils.py index 9190778a1..bc4081666 100644 --- a/vndk/snapshot/utils.py +++ b/vndk/snapshot/utils.py @@ -17,6 +17,7 @@ """Utility functions for VNDK snapshot.""" import glob +import logging import os import re import subprocess @@ -34,6 +35,24 @@ NOTICE_FILES_DIR_NAME = 'NOTICE_FILES' NOTICE_FILES_DIR_PATH = os.path.join(COMMON_DIR_PATH, NOTICE_FILES_DIR_NAME) +def logger(name): + return logging.getLogger(name) + + +def set_logging_config(verbose_level): + verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) + verbosity = min(verbose_level, 2) + logging.basicConfig( + format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', + level=verbose_map[verbosity]) + + +def check_call(cmd, logger=None): + logger = logger or logging + logger.debug('Running `{}`'.format(' '.join(cmd))) + subprocess.check_call(cmd) + + def get_android_build_top(): ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') if not ANDROID_BUILD_TOP: @@ -129,5 +148,4 @@ def fetch_artifact(branch, build, pattern, destination='.'): fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid', build, pattern, destination ] - print 'Running `{}`'.format(' '.join(cmd)) - subprocess.check_call(cmd) + check_call(cmd, logger(__name__))