Merge "Implement more features to vendor snapshot script"

vendor_snapshot/update.py

@@ -30,18 +30,23 @@ import json
 
 INDENT = ' ' * 4
 
+
 def get_notice_path(module_name):
-    return os.path.join('NOTICE_FILES', module_name+'.txt')
+    return os.path.join('NOTICE_FILES', module_name + '.txt')
+
 
 def get_target_arch(json_rel_path):
     return json_rel_path.split('/')[0]
 
+
 def get_arch(json_rel_path):
     return json_rel_path.split('/')[1].split('-')[1]
 
+
 def get_variation(json_rel_path):
     return json_rel_path.split('/')[2]
 
+
 # convert .bp prop dictionary to .bp prop string
 def gen_bp_prop(prop, ind):
     bp = ''
@@ -54,9 +59,9 @@ def gen_bp_prop(prop, ind):
             if len(val) == 0:
                 continue
 
-        bp += ind + key + ": "
+        bp += ind + key + ': '
         if type(val) == bool:
-            bp += "true,\n" if val else "false,\n"
+            bp += 'true,\n' if val else 'false,\n'
         elif type(val) == str:
             bp += '"%s",\n' % val
         elif type(val) == list:
@@ -72,6 +77,7 @@ def gen_bp_prop(prop, ind):
             raise TypeError('unsupported type %s for gen_bp_prop' % type(val))
     return bp
 
+
 # Remove non-existent dirs from given list. Emits warning for such dirs.
 def remove_invalid_dirs(paths, bp_dir, module_name):
     ret = []
@@ -79,25 +85,26 @@ def remove_invalid_dirs(paths, bp_dir, module_name):
         if os.path.isdir(os.path.join(bp_dir, path)):
             ret.append(path)
         else:
-            logging.warning(
-                'Dir "%s" of module "%s" does not exist' % (path, module_name))
+            logging.warning('Dir "%s" of module "%s" does not exist', path,
+                            module_name)
     return ret
 
+
 JSON_TO_BP = {
-    'ModuleName':          'name',
+    'ModuleName': 'name',
     'RelativeInstallPath': 'relative_install_path',
-    'ExportedDirs':        'export_include_dirs',
-    'ExportedSystemDirs':  'export_system_include_dirs',
-    'ExportedFlags':       'export_flags',
-    'Sanitize':            'sanitize',
-    'SanitizeMinimalDep':  'sanitize_minimal_dep',
-    'SanitizeUbsanDep':    'sanitize_ubsan_dep',
-    'Symlinks':            'symlinks',
-    'InitRc':              'init_rc',
-    'VintfFragments':      'vintf_fragments',
-    'SharedLibs':          'shared_libs',
-    'RuntimeLibs':         'runtime_libs',
-    'Required':            'required',
+    'ExportedDirs': 'export_include_dirs',
+    'ExportedSystemDirs': 'export_system_include_dirs',
+    'ExportedFlags': 'export_flags',
+    'Sanitize': 'sanitize',
+    'SanitizeMinimalDep': 'sanitize_minimal_dep',
+    'SanitizeUbsanDep': 'sanitize_ubsan_dep',
+    'Symlinks': 'symlinks',
+    'InitRc': 'init_rc',
+    'VintfFragments': 'vintf_fragments',
+    'SharedLibs': 'shared_libs',
+    'RuntimeLibs': 'runtime_libs',
+    'Required': 'required',
 }
 
 SANITIZER_VARIANT_PROPS = {
@@ -109,6 +116,7 @@ SANITIZER_VARIANT_PROPS = {
     'src',
 }
 
+
 # Converts parsed json dictionary (which is intermediate) to Android.bp prop
 # dictionary. This validates paths such as include directories and init_rc
 # files while converting.
@@ -131,11 +139,12 @@ def convert_json_to_bp_prop(json_path, bp_dir):
         if key in JSON_TO_BP:
             ret[JSON_TO_BP[key]] = prop[key]
         else:
-            logging.warning(
-                'Unknown prop "%s" of module "%s"' % (key, module_name))
+            logging.warning('Unknown prop "%s" of module "%s"', key,
+                            module_name)
 
     return ret
 
+
 def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
     prop = {
         # These three are common for all snapshot modules.
@@ -154,7 +163,7 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
                 common_prop[k] = arch_props[arch][k]
             continue
         for k in list(common_prop.keys()):
-            if not k in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
+            if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
                 del common_prop[k]
 
     # Some keys has to be arch_props to prevent 32-bit only modules from being
@@ -174,7 +183,7 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
         # Record stem for executable binary snapshots.
         # We don't check existence of 'src'; src must exist for executables
         if variation == 'binary':
-            if '64' in arch: # arm64, x86_64
+            if '64' in arch:  # arm64, x86_64
                 stem64 = os.path.basename(arch_props[arch]['src'])
             else:
                 stem32 = os.path.basename(arch_props[arch]['src'])
@@ -198,30 +207,8 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
     bp += '}\n\n'
     return bp
 
-def get_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'snapshot_version',
-        type=int,
-        help='Vendor snapshot version to install, e.g. "30".')
-    parser.add_argument(
-        '-v',
-        '--verbose',
-        action='count',
-        default=0,
-        help='Increase output verbosity, e.g. "-v", "-vv".')
-    return parser.parse_args()
-
-def main():
-    """Program entry point."""
-    args = get_args()
-    verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
-    verbosity = min(args.verbose, 2)
-    logging.basicConfig(
-        format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
-        level=verbose_map[verbosity])
-    install_dir = os.path.join('prebuilts', 'vendor', 'v'+str(args.snapshot_version))
 
+def gen_bp_files(install_dir, snapshot_version):
     # props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json
     props = dict()
 
@@ -250,11 +237,11 @@ def main():
 
             if variation != 'header':
                 prop['src'] = os.path.relpath(
-                    rel_path[:-5], # removing .json
+                    rel_path[:-5],  # removing .json
                     target_arch)
 
             module_name = prop['name']
-            notice_path = 'NOTICE_FILES/' + module_name + ".txt"
+            notice_path = 'NOTICE_FILES/' + module_name + '.txt'
             if os.path.exists(os.path.join(bp_dir, notice_path)):
                 prop['notice'] = notice_path
 
@@ -262,7 +249,7 @@ def main():
             if 'sanitize' in prop:
                 sanitizer_type = prop['sanitize']
                 # module_name is {name}.{sanitizer_type}; trim sanitizer_type
-                module_name = module_name[:-len(sanitizer_type)-1]
+                module_name = module_name[:-len(sanitizer_type) - 1]
                 # Only leave props for the sanitize variant
                 for k in list(prop.keys()):
                     if not k in SANITIZER_VARIANT_PROPS:
@@ -282,15 +269,161 @@ def main():
         bp_dir = os.path.join(install_dir, target_arch)
         for variation in props[target_arch]:
             for name in props[target_arch][variation]:
-                androidbp += gen_bp_module(
-                    variation,
-                    name,
-                    args.snapshot_version,
-                    target_arch,
-                    props[target_arch][variation][name],
-                    bp_dir)
+                androidbp += gen_bp_module(variation, name, snapshot_version,
+                                           target_arch,
+                                           props[target_arch][variation][name],
+                                           bp_dir)
         with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f:
+            logging.info('Generating Android.bp to: {}'.format(f.name))
             f.write(androidbp)
 
+
+def check_call(cmd):
+    logging.debug('Running `{}`'.format(' '.join(cmd)))
+    subprocess.check_call(cmd)
+
+
+def fetch_artifact(branch, build, target, pattern, destination):
+    """Fetches build artifacts from Android Build server.
+
+    Args:
+      branch: string, branch to pull build artifacts from
+      build: string, build number to pull build artifacts from
+      target: string, target name to pull build artifacts from
+      pattern: string, pattern of build artifact file name
+      destination: string, destination to pull build artifact to
+    """
+    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
+    cmd = [
+        fetch_artifact_path, '--branch', branch, '--target', target, '--bid',
+        build, pattern, destination
+    ]
+    check_call(cmd)
+
+def install_artifacts(branch, build, target, local_dir, install_dir):
+    """Installs vendor snapshot build artifacts to {install_dir}/v{version}.
+
+    1) Fetch build artifacts from Android Build server or from local_dir
+    2) Unzip build artifacts
+
+    Args:
+      branch: string or None, branch name of build artifacts
+      build: string or None, build number of build artifacts
+      target: string or None, target name of build artifacts
+      local_dir: string or None, local dir to pull artifacts from
+      install_dir: string, directory to install vendor snapshot
+      temp_artifact_dir: string, temp directory to hold build artifacts fetched
+        from Android Build server. For 'local' option, is set to None.
+    """
+    artifact_pattern = 'vendor-*.zip'
+
+    def unzip_artifacts(artifact_dir):
+        artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
+        for artifact in artifacts:
+            logging.info('Unzipping Vendor snapshot: {}'.format(artifact))
+            check_call(['unzip', '-qn', artifact, '-d', install_dir])
+
+    if branch and build and target:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logging.info(
+                'Fetching {pattern} from {branch} (bid: {build}, target: {target})'
+                .format(
+                    pattern=artifact_pattern,
+                    branch=branch,
+                    build=build,
+                    target=target))
+            fetch_artifact(branch, build, target, artifact_pattern, tmpdir)
+            unzip_artifacts(tmpdir)
+    elif local_dir:
+        logging.info('Fetching local VNDK snapshot from {}'.format(local_dir))
+        unzip_artifacts(local_dir)
+    else:
+        raise RuntimeError('Neither local nor remote fetch information given.')
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'snapshot_version',
+        type=int,
+        help='Vendor snapshot version to install, e.g. "30".')
+    parser.add_argument('--branch', help='Branch to pull build from.')
+    parser.add_argument('--build', help='Build number to pull.')
+    parser.add_argument('--target', help='Target to pull.')
+    parser.add_argument(
+        '--local',
+        help=('Fetch local vendor snapshot artifacts from specified local '
+              'directory instead of Android Build server. '
+              'Example: --local /path/to/local/dir'))
+    parser.add_argument(
+        '--install-dir',
+        help=(
+            'Base directory to which vendor snapshot artifacts are installed. '
+            'Example: --install-dir vendor/<company name>/vendor_snapshot/v30'))
+
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='count',
+        default=0,
+        help='Increase output verbosity, e.g. "-v", "-vv".')
+    return parser.parse_args()
+
+
+def main():
+    """Program entry point."""
+    args = get_args()
+
+    local = None
+    if args.local:
+        local = os.path.expanduser(args.local)
+
+    if local:
+        if args.build or args.branch or args.target:
+            raise ValueError(
+                'When --local option is set, --branch, --build or --target cannot be '
+                'specified.')
+        elif not os.path.isdir(local):
+            raise RuntimeError(
+                'The specified local directory, {}, does not exist.'.format(
+                    local))
+    else:
+        if not (args.build and args.branch and args.target):
+            raise ValueError(
+                'Please provide --branch, --build and --target. Or set --local '
+                'option.')
+
+    if not args.install_dir:
+        raise ValueError('Please provide --install-dir option.')
+
+    snapshot_version = args.snapshot_version
+
+    verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
+    verbosity = min(args.verbose, 2)
+    logging.basicConfig(
+        format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
+        level=verbose_map[verbosity])
+
+    install_dir = os.path.expanduser(args.install_dir)
+    if os.path.exists(install_dir):
+        resp = input('Directory {} already exists. IT WILL BE REMOVED.\n'
+                     'Are you sure? (yes/no): '.format(install_dir))
+        if resp == 'yes':
+            logging.info('Removing {}'.format(install_dir))
+            check_call(['rm', '-rf', install_dir])
+        elif resp == 'no':
+            logging.info('Cancelled snapshot install.')
+            return
+        else:
+            raise ValueError('Did not understand: ' + resp)
+    check_call(['mkdir', '-p', install_dir])
+
+    install_artifacts(
+        branch=args.branch,
+        build=args.build,
+        target=args.target,
+        local_dir=local,
+        install_dir=install_dir)
+    gen_bp_files(install_dir, snapshot_version)
+
 if __name__ == '__main__':
     main()