diff --git a/vendor_snapshot/update.py b/vendor_snapshot/update.py index 429949997..71e2ffdf6 100644 --- a/vendor_snapshot/update.py +++ b/vendor_snapshot/update.py @@ -30,18 +30,23 @@ import json INDENT = ' ' * 4 + def get_notice_path(module_name): - return os.path.join('NOTICE_FILES', module_name+'.txt') + return os.path.join('NOTICE_FILES', module_name + '.txt') + def get_target_arch(json_rel_path): return json_rel_path.split('/')[0] + def get_arch(json_rel_path): return json_rel_path.split('/')[1].split('-')[1] + def get_variation(json_rel_path): return json_rel_path.split('/')[2] + # convert .bp prop dictionary to .bp prop string def gen_bp_prop(prop, ind): bp = '' @@ -54,9 +59,9 @@ def gen_bp_prop(prop, ind): if len(val) == 0: continue - bp += ind + key + ": " + bp += ind + key + ': ' if type(val) == bool: - bp += "true,\n" if val else "false,\n" + bp += 'true,\n' if val else 'false,\n' elif type(val) == str: bp += '"%s",\n' % val elif type(val) == list: @@ -72,6 +77,7 @@ def gen_bp_prop(prop, ind): raise TypeError('unsupported type %s for gen_bp_prop' % type(val)) return bp + # Remove non-existent dirs from given list. Emits warning for such dirs. def remove_invalid_dirs(paths, bp_dir, module_name): ret = [] @@ -79,25 +85,26 @@ def remove_invalid_dirs(paths, bp_dir, module_name): if os.path.isdir(os.path.join(bp_dir, path)): ret.append(path) else: - logging.warning( - 'Dir "%s" of module "%s" does not exist' % (path, module_name)) + logging.warning('Dir "%s" of module "%s" does not exist', path, + module_name) return ret + JSON_TO_BP = { - 'ModuleName': 'name', + 'ModuleName': 'name', 'RelativeInstallPath': 'relative_install_path', - 'ExportedDirs': 'export_include_dirs', - 'ExportedSystemDirs': 'export_system_include_dirs', - 'ExportedFlags': 'export_flags', - 'Sanitize': 'sanitize', - 'SanitizeMinimalDep': 'sanitize_minimal_dep', - 'SanitizeUbsanDep': 'sanitize_ubsan_dep', - 'Symlinks': 'symlinks', - 'InitRc': 'init_rc', - 'VintfFragments': 'vintf_fragments', - 'SharedLibs': 'shared_libs', - 'RuntimeLibs': 'runtime_libs', - 'Required': 'required', + 'ExportedDirs': 'export_include_dirs', + 'ExportedSystemDirs': 'export_system_include_dirs', + 'ExportedFlags': 'export_flags', + 'Sanitize': 'sanitize', + 'SanitizeMinimalDep': 'sanitize_minimal_dep', + 'SanitizeUbsanDep': 'sanitize_ubsan_dep', + 'Symlinks': 'symlinks', + 'InitRc': 'init_rc', + 'VintfFragments': 'vintf_fragments', + 'SharedLibs': 'shared_libs', + 'RuntimeLibs': 'runtime_libs', + 'Required': 'required', } SANITIZER_VARIANT_PROPS = { @@ -109,6 +116,7 @@ SANITIZER_VARIANT_PROPS = { 'src', } + # Converts parsed json dictionary (which is intermediate) to Android.bp prop # dictionary. This validates paths such as include directories and init_rc # files while converting. @@ -131,11 +139,12 @@ def convert_json_to_bp_prop(json_path, bp_dir): if key in JSON_TO_BP: ret[JSON_TO_BP[key]] = prop[key] else: - logging.warning( - 'Unknown prop "%s" of module "%s"' % (key, module_name)) + logging.warning('Unknown prop "%s" of module "%s"', key, + module_name) return ret + def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir): prop = { # These three are common for all snapshot modules. @@ -154,7 +163,7 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir): common_prop[k] = arch_props[arch][k] continue for k in list(common_prop.keys()): - if not k in arch_props[arch] or common_prop[k] != arch_props[arch][k]: + if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]: del common_prop[k] # Some keys has to be arch_props to prevent 32-bit only modules from being @@ -174,7 +183,7 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir): # Record stem for executable binary snapshots. # We don't check existence of 'src'; src must exist for executables if variation == 'binary': - if '64' in arch: # arm64, x86_64 + if '64' in arch: # arm64, x86_64 stem64 = os.path.basename(arch_props[arch]['src']) else: stem32 = os.path.basename(arch_props[arch]['src']) @@ -198,30 +207,8 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir): bp += '}\n\n' return bp -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'snapshot_version', - type=int, - help='Vendor snapshot version to install, e.g. "30".') - parser.add_argument( - '-v', - '--verbose', - action='count', - default=0, - help='Increase output verbosity, e.g. "-v", "-vv".') - return parser.parse_args() - -def main(): - """Program entry point.""" - args = get_args() - verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) - verbosity = min(args.verbose, 2) - logging.basicConfig( - format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', - level=verbose_map[verbosity]) - install_dir = os.path.join('prebuilts', 'vendor', 'v'+str(args.snapshot_version)) +def gen_bp_files(install_dir, snapshot_version): # props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json props = dict() @@ -250,11 +237,11 @@ def main(): if variation != 'header': prop['src'] = os.path.relpath( - rel_path[:-5], # removing .json + rel_path[:-5], # removing .json target_arch) module_name = prop['name'] - notice_path = 'NOTICE_FILES/' + module_name + ".txt" + notice_path = 'NOTICE_FILES/' + module_name + '.txt' if os.path.exists(os.path.join(bp_dir, notice_path)): prop['notice'] = notice_path @@ -262,7 +249,7 @@ def main(): if 'sanitize' in prop: sanitizer_type = prop['sanitize'] # module_name is {name}.{sanitizer_type}; trim sanitizer_type - module_name = module_name[:-len(sanitizer_type)-1] + module_name = module_name[:-len(sanitizer_type) - 1] # Only leave props for the sanitize variant for k in list(prop.keys()): if not k in SANITIZER_VARIANT_PROPS: @@ -282,15 +269,161 @@ def main(): bp_dir = os.path.join(install_dir, target_arch) for variation in props[target_arch]: for name in props[target_arch][variation]: - androidbp += gen_bp_module( - variation, - name, - args.snapshot_version, - target_arch, - props[target_arch][variation][name], - bp_dir) + androidbp += gen_bp_module(variation, name, snapshot_version, + target_arch, + props[target_arch][variation][name], + bp_dir) with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f: + logging.info('Generating Android.bp to: {}'.format(f.name)) f.write(androidbp) + +def check_call(cmd): + logging.debug('Running `{}`'.format(' '.join(cmd))) + subprocess.check_call(cmd) + + +def fetch_artifact(branch, build, target, pattern, destination): + """Fetches build artifacts from Android Build server. + + Args: + branch: string, branch to pull build artifacts from + build: string, build number to pull build artifacts from + target: string, target name to pull build artifacts from + pattern: string, pattern of build artifact file name + destination: string, destination to pull build artifact to + """ + fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' + cmd = [ + fetch_artifact_path, '--branch', branch, '--target', target, '--bid', + build, pattern, destination + ] + check_call(cmd) + +def install_artifacts(branch, build, target, local_dir, install_dir): + """Installs vendor snapshot build artifacts to {install_dir}/v{version}. + + 1) Fetch build artifacts from Android Build server or from local_dir + 2) Unzip build artifacts + + Args: + branch: string or None, branch name of build artifacts + build: string or None, build number of build artifacts + target: string or None, target name of build artifacts + local_dir: string or None, local dir to pull artifacts from + install_dir: string, directory to install vendor snapshot + temp_artifact_dir: string, temp directory to hold build artifacts fetched + from Android Build server. For 'local' option, is set to None. + """ + artifact_pattern = 'vendor-*.zip' + + def unzip_artifacts(artifact_dir): + artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) + for artifact in artifacts: + logging.info('Unzipping Vendor snapshot: {}'.format(artifact)) + check_call(['unzip', '-qn', artifact, '-d', install_dir]) + + if branch and build and target: + with tempfile.TemporaryDirectory() as tmpdir: + logging.info( + 'Fetching {pattern} from {branch} (bid: {build}, target: {target})' + .format( + pattern=artifact_pattern, + branch=branch, + build=build, + target=target)) + fetch_artifact(branch, build, target, artifact_pattern, tmpdir) + unzip_artifacts(tmpdir) + elif local_dir: + logging.info('Fetching local VNDK snapshot from {}'.format(local_dir)) + unzip_artifacts(local_dir) + else: + raise RuntimeError('Neither local nor remote fetch information given.') + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'snapshot_version', + type=int, + help='Vendor snapshot version to install, e.g. "30".') + parser.add_argument('--branch', help='Branch to pull build from.') + parser.add_argument('--build', help='Build number to pull.') + parser.add_argument('--target', help='Target to pull.') + parser.add_argument( + '--local', + help=('Fetch local vendor snapshot artifacts from specified local ' + 'directory instead of Android Build server. ' + 'Example: --local /path/to/local/dir')) + parser.add_argument( + '--install-dir', + help=( + 'Base directory to which vendor snapshot artifacts are installed. ' + 'Example: --install-dir vendor//vendor_snapshot/v30')) + + parser.add_argument( + '-v', + '--verbose', + action='count', + default=0, + help='Increase output verbosity, e.g. "-v", "-vv".') + return parser.parse_args() + + +def main(): + """Program entry point.""" + args = get_args() + + local = None + if args.local: + local = os.path.expanduser(args.local) + + if local: + if args.build or args.branch or args.target: + raise ValueError( + 'When --local option is set, --branch, --build or --target cannot be ' + 'specified.') + elif not os.path.isdir(local): + raise RuntimeError( + 'The specified local directory, {}, does not exist.'.format( + local)) + else: + if not (args.build and args.branch and args.target): + raise ValueError( + 'Please provide --branch, --build and --target. Or set --local ' + 'option.') + + if not args.install_dir: + raise ValueError('Please provide --install-dir option.') + + snapshot_version = args.snapshot_version + + verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) + verbosity = min(args.verbose, 2) + logging.basicConfig( + format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', + level=verbose_map[verbosity]) + + install_dir = os.path.expanduser(args.install_dir) + if os.path.exists(install_dir): + resp = input('Directory {} already exists. IT WILL BE REMOVED.\n' + 'Are you sure? (yes/no): '.format(install_dir)) + if resp == 'yes': + logging.info('Removing {}'.format(install_dir)) + check_call(['rm', '-rf', install_dir]) + elif resp == 'no': + logging.info('Cancelled snapshot install.') + return + else: + raise ValueError('Did not understand: ' + resp) + check_call(['mkdir', '-p', install_dir]) + + install_artifacts( + branch=args.branch, + build=args.build, + target=args.target, + local_dir=local, + install_dir=install_dir) + gen_bp_files(install_dir, snapshot_version) + if __name__ == '__main__': main()