3bdbd34b9a
In cases where not all the variants of a library are included in the generated vendor snapshot, be sure to include the module name in the props so that it can be factored out later. Conversely, be sure to compute the notice filename from the de-sanitized module name so so that it can also be factored out later. Bug: 171413810 Bug: 157967325 Test: Verify that full snapshot Android.bp unchanged. Test: Verify correct syntax for filtered snapshot. Change-Id: I5725e17b19907d6a5063823e8b99bd0517c7c144
473 lines
16 KiB
Python
473 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2020 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
"""Unzips and installs the vendor snapshot."""
|
|
|
|
import argparse
|
|
import glob
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import json
|
|
|
|
INDENT = ' ' * 4
|
|
|
|
|
|
def get_notice_path(module_name):
|
|
return os.path.join('NOTICE_FILES', module_name + '.txt')
|
|
|
|
|
|
def get_target_arch(json_rel_path):
|
|
return json_rel_path.split('/')[0]
|
|
|
|
|
|
def get_arch(json_rel_path):
|
|
return json_rel_path.split('/')[1].split('-')[1]
|
|
|
|
|
|
def get_variation(json_rel_path):
|
|
return json_rel_path.split('/')[2]
|
|
|
|
|
|
# convert .bp prop dictionary to .bp prop string
|
|
def gen_bp_prop(prop, ind):
|
|
bp = ''
|
|
for key in prop:
|
|
val = prop[key]
|
|
|
|
# Skip empty list or dict, rather than printing empty prop like
|
|
# "key: []," or "key: {},"
|
|
if type(val) == list or type(val) == dict:
|
|
if len(val) == 0:
|
|
continue
|
|
|
|
bp += ind + key + ': '
|
|
if type(val) == bool:
|
|
bp += 'true,\n' if val else 'false,\n'
|
|
elif type(val) == str:
|
|
bp += '"%s",\n' % val
|
|
elif type(val) == list:
|
|
bp += '[\n'
|
|
for elem in val:
|
|
bp += ind + INDENT + '"%s",\n' % elem
|
|
bp += ind + '],\n'
|
|
elif type(val) == dict:
|
|
bp += '{\n'
|
|
bp += gen_bp_prop(val, ind + INDENT)
|
|
bp += ind + '},\n'
|
|
else:
|
|
raise TypeError('unsupported type %s for gen_bp_prop' % type(val))
|
|
return bp
|
|
|
|
|
|
# Remove non-existent dirs from given list. Emits warning for such dirs.
|
|
def remove_invalid_dirs(paths, bp_dir, module_name):
|
|
ret = []
|
|
for path in paths:
|
|
if os.path.isdir(os.path.join(bp_dir, path)):
|
|
ret.append(path)
|
|
else:
|
|
logging.warning('Dir "%s" of module "%s" does not exist', path,
|
|
module_name)
|
|
return ret
|
|
|
|
|
|
JSON_TO_BP = {
|
|
'ModuleName': 'name',
|
|
'RelativeInstallPath': 'relative_install_path',
|
|
'ExportedDirs': 'export_include_dirs',
|
|
'ExportedSystemDirs': 'export_system_include_dirs',
|
|
'ExportedFlags': 'export_flags',
|
|
'Sanitize': 'sanitize',
|
|
'SanitizeMinimalDep': 'sanitize_minimal_dep',
|
|
'SanitizeUbsanDep': 'sanitize_ubsan_dep',
|
|
'Symlinks': 'symlinks',
|
|
'InitRc': 'init_rc',
|
|
'VintfFragments': 'vintf_fragments',
|
|
'SharedLibs': 'shared_libs',
|
|
'RuntimeLibs': 'runtime_libs',
|
|
'Required': 'required',
|
|
}
|
|
|
|
SANITIZER_VARIANT_PROPS = {
|
|
'export_include_dirs',
|
|
'export_system_include_dirs',
|
|
'export_flags',
|
|
'sanitize_minimal_dep',
|
|
'sanitize_ubsan_dep',
|
|
'src',
|
|
}
|
|
|
|
|
|
# Converts parsed json dictionary (which is intermediate) to Android.bp prop
|
|
# dictionary. This validates paths such as include directories and init_rc
|
|
# files while converting.
|
|
def convert_json_to_bp_prop(json_path, bp_dir):
|
|
prop = json.load(json_path)
|
|
ret = {}
|
|
|
|
module_name = prop['ModuleName']
|
|
ret['name'] = module_name
|
|
|
|
# Soong will complain about non-existing paths on Android.bp. There might
|
|
# be missing files among generated header files, so check all exported
|
|
# directories and filter out invalid ones. Emits warning for such dirs.
|
|
# TODO: fix soong to track all generated headers correctly
|
|
for key in {'ExportedDirs', 'ExportedSystemDirs'}:
|
|
if key in prop:
|
|
prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name)
|
|
|
|
for key in prop:
|
|
if key in JSON_TO_BP:
|
|
ret[JSON_TO_BP[key]] = prop[key]
|
|
else:
|
|
logging.warning('Unknown prop "%s" of module "%s"', key,
|
|
module_name)
|
|
|
|
return ret
|
|
|
|
|
|
def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
|
|
prop = {
|
|
# These three are common for all snapshot modules.
|
|
'version': str(version),
|
|
'target_arch': target_arch,
|
|
'vendor': True,
|
|
'arch': {},
|
|
}
|
|
|
|
# Factor out common prop among architectures to minimize Android.bp.
|
|
common_prop = None
|
|
for arch in arch_props:
|
|
if common_prop is None:
|
|
common_prop = dict()
|
|
for k in arch_props[arch]:
|
|
common_prop[k] = arch_props[arch][k]
|
|
continue
|
|
for k in list(common_prop.keys()):
|
|
if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
|
|
del common_prop[k]
|
|
|
|
# Some keys has to be arch_props to prevent 32-bit only modules from being
|
|
# used as 64-bit modules, and vice versa.
|
|
for arch_prop_key in ['src', 'cfi']:
|
|
if arch_prop_key in common_prop:
|
|
del common_prop[arch_prop_key]
|
|
prop.update(common_prop)
|
|
|
|
stem32 = stem64 = ''
|
|
|
|
for arch in arch_props:
|
|
for k in common_prop:
|
|
if k in arch_props[arch]:
|
|
del arch_props[arch][k]
|
|
prop['arch'][arch] = arch_props[arch]
|
|
# Record stem for executable binary snapshots.
|
|
# We don't check existence of 'src'; src must exist for executables
|
|
if variation == 'binary':
|
|
if '64' in arch: # arm64, x86_64
|
|
stem64 = os.path.basename(arch_props[arch]['src'])
|
|
else:
|
|
stem32 = os.path.basename(arch_props[arch]['src'])
|
|
|
|
# For binary snapshots, compile_multilib must be assigned to 'both'
|
|
# in order to install both. Prefer 64bit if their stem collide and
|
|
# installing both is impossible
|
|
if variation == 'binary':
|
|
if stem32 and stem64:
|
|
if stem32 == stem64:
|
|
prop['compile_multilib'] = 'first'
|
|
else:
|
|
prop['compile_multilib'] = 'both'
|
|
elif stem32:
|
|
prop['compile_multilib'] = '32'
|
|
elif stem64:
|
|
prop['compile_multilib'] = '64'
|
|
|
|
bp = 'vendor_snapshot_%s {\n' % variation
|
|
bp += gen_bp_prop(prop, INDENT)
|
|
bp += '}\n\n'
|
|
return bp
|
|
|
|
|
|
def build_props(install_dir):
|
|
# props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json
|
|
props = dict()
|
|
|
|
# {target_arch}/{arch}/{variation}/{module}.json
|
|
for root, _, files in os.walk(install_dir, followlinks = True):
|
|
for file_name in sorted(files):
|
|
if not file_name.endswith('.json'):
|
|
continue
|
|
full_path = os.path.join(root, file_name)
|
|
rel_path = os.path.relpath(full_path, install_dir)
|
|
|
|
target_arch = get_target_arch(rel_path)
|
|
arch = get_arch(rel_path)
|
|
variation = get_variation(rel_path)
|
|
bp_dir = os.path.join(install_dir, target_arch)
|
|
|
|
if not target_arch in props:
|
|
props[target_arch] = dict()
|
|
if not variation in props[target_arch]:
|
|
props[target_arch][variation] = dict()
|
|
|
|
with open(full_path, 'r') as f:
|
|
prop = convert_json_to_bp_prop(f, bp_dir)
|
|
# Remove .json after parsing?
|
|
# os.unlink(full_path)
|
|
|
|
if variation != 'header':
|
|
prop['src'] = os.path.relpath(
|
|
rel_path[:-5], # removing .json
|
|
target_arch)
|
|
|
|
module_name = prop['name']
|
|
|
|
# Is this sanitized variant?
|
|
if 'sanitize' in prop:
|
|
sanitizer_type = prop['sanitize']
|
|
# module_name is {name}.{sanitizer_type}; trim sanitizer_type
|
|
module_name = module_name[:-len(sanitizer_type) - 1]
|
|
# Only leave props for the sanitize variant
|
|
for k in list(prop.keys()):
|
|
if not k in SANITIZER_VARIANT_PROPS:
|
|
del prop[k]
|
|
prop = {'name': module_name, sanitizer_type: prop}
|
|
|
|
notice_path = 'NOTICE_FILES/' + module_name + '.txt'
|
|
if os.path.exists(os.path.join(bp_dir, notice_path)):
|
|
prop['notice'] = notice_path
|
|
|
|
variation_dict = props[target_arch][variation]
|
|
if not module_name in variation_dict:
|
|
variation_dict[module_name] = dict()
|
|
if not arch in variation_dict[module_name]:
|
|
variation_dict[module_name][arch] = prop
|
|
else:
|
|
variation_dict[module_name][arch].update(prop)
|
|
|
|
return props
|
|
|
|
|
|
def gen_bp_files(install_dir, snapshot_version):
|
|
props = build_props(install_dir)
|
|
|
|
for target_arch in sorted(props):
|
|
androidbp = ''
|
|
bp_dir = os.path.join(install_dir, target_arch)
|
|
for variation in sorted(props[target_arch]):
|
|
for name in sorted(props[target_arch][variation]):
|
|
androidbp += gen_bp_module(variation, name, snapshot_version,
|
|
target_arch,
|
|
props[target_arch][variation][name],
|
|
bp_dir)
|
|
with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f:
|
|
logging.info('Generating Android.bp to: {}'.format(f.name))
|
|
f.write(androidbp)
|
|
|
|
|
|
def check_call(cmd):
|
|
logging.debug('Running `{}`'.format(' '.join(cmd)))
|
|
subprocess.check_call(cmd)
|
|
|
|
|
|
def fetch_artifact(branch, build, target, pattern, destination):
|
|
"""Fetches build artifacts from Android Build server.
|
|
|
|
Args:
|
|
branch: string, branch to pull build artifacts from
|
|
build: string, build number to pull build artifacts from
|
|
target: string, target name to pull build artifacts from
|
|
pattern: string, pattern of build artifact file name
|
|
destination: string, destination to pull build artifact to
|
|
"""
|
|
fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
|
|
cmd = [
|
|
fetch_artifact_path, '--branch', branch, '--target', target, '--bid',
|
|
build, pattern, destination
|
|
]
|
|
check_call(cmd)
|
|
|
|
def install_artifacts(branch, build, target, local_dir, symlink, install_dir):
|
|
"""Installs vendor snapshot build artifacts to {install_dir}/v{version}.
|
|
|
|
1) Fetch build artifacts from Android Build server or from local_dir
|
|
2) Unzip or create symlinks to build artifacts
|
|
|
|
Args:
|
|
branch: string or None, branch name of build artifacts
|
|
build: string or None, build number of build artifacts
|
|
target: string or None, target name of build artifacts
|
|
local_dir: string or None, local dir to pull artifacts from
|
|
symlink: boolean, whether to use symlinks instead of unzipping the
|
|
vendor snapshot zip
|
|
install_dir: string, directory to install vendor snapshot
|
|
temp_artifact_dir: string, temp directory to hold build artifacts fetched
|
|
from Android Build server. For 'local' option, is set to None.
|
|
"""
|
|
artifact_pattern = 'vendor-*.zip'
|
|
|
|
def unzip_artifacts(artifact_dir):
|
|
artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
|
|
for artifact in artifacts:
|
|
logging.info('Unzipping Vendor snapshot: {}'.format(artifact))
|
|
check_call(['unzip', '-qn', artifact, '-d', install_dir])
|
|
|
|
if branch and build and target:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
logging.info(
|
|
'Fetching {pattern} from {branch} (bid: {build}, target: {target})'
|
|
.format(
|
|
pattern=artifact_pattern,
|
|
branch=branch,
|
|
build=build,
|
|
target=target))
|
|
fetch_artifact(branch, build, target, artifact_pattern, tmpdir)
|
|
unzip_artifacts(tmpdir)
|
|
elif local_dir:
|
|
if symlink:
|
|
# This assumes local_dir is the location of vendor-snapshot in the
|
|
# build (e.g., out/soong/vendor-snapshot).
|
|
#
|
|
# Create the first level as proper directories and the next level
|
|
# as symlinks.
|
|
for item1 in os.listdir(local_dir):
|
|
dest_dir = os.path.join(install_dir, item1)
|
|
src_dir = os.path.join(local_dir, item1)
|
|
if os.path.isdir(src_dir):
|
|
check_call(['mkdir', '-p', dest_dir])
|
|
# Create symlinks.
|
|
for item2 in os.listdir(src_dir):
|
|
src_item = os.path.join(src_dir, item2)
|
|
logging.info('Creating symlink from {} in {}'.format(
|
|
src_item, dest_dir))
|
|
os.symlink(src_item, os.path.join(dest_dir, item2))
|
|
else:
|
|
logging.info('Fetching local VNDK snapshot from {}'.format(
|
|
local_dir))
|
|
unzip_artifacts(local_dir)
|
|
else:
|
|
raise RuntimeError('Neither local nor remote fetch information given.')
|
|
|
|
def get_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'snapshot_version',
|
|
type=int,
|
|
help='Vendor snapshot version to install, e.g. "30".')
|
|
parser.add_argument('--branch', help='Branch to pull build from.')
|
|
parser.add_argument('--build', help='Build number to pull.')
|
|
parser.add_argument('--target', help='Target to pull.')
|
|
parser.add_argument(
|
|
'--local',
|
|
help=('Fetch local vendor snapshot artifacts from specified local '
|
|
'directory instead of Android Build server. '
|
|
'Example: --local /path/to/local/dir'))
|
|
parser.add_argument(
|
|
'--symlink',
|
|
action='store_true',
|
|
help='Use symlinks instead of unzipping vendor snapshot zip')
|
|
parser.add_argument(
|
|
'--install-dir',
|
|
help=(
|
|
'Base directory to which vendor snapshot artifacts are installed. '
|
|
'Example: --install-dir vendor/<company name>/vendor_snapshot/v30'))
|
|
parser.add_argument(
|
|
'--overwrite',
|
|
action='store_true',
|
|
help=(
|
|
'If provided, does not ask before overwriting the install-dir.'))
|
|
|
|
parser.add_argument(
|
|
'-v',
|
|
'--verbose',
|
|
action='count',
|
|
default=0,
|
|
help='Increase output verbosity, e.g. "-v", "-vv".')
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
"""Program entry point."""
|
|
args = get_args()
|
|
|
|
local = None
|
|
if args.local:
|
|
local = os.path.expanduser(args.local)
|
|
|
|
if local:
|
|
if args.build or args.branch or args.target:
|
|
raise ValueError(
|
|
'When --local option is set, --branch, --build or --target cannot be '
|
|
'specified.')
|
|
elif not os.path.isdir(local):
|
|
raise RuntimeError(
|
|
'The specified local directory, {}, does not exist.'.format(
|
|
local))
|
|
else:
|
|
if not (args.build and args.branch and args.target):
|
|
raise ValueError(
|
|
'Please provide --branch, --build and --target. Or set --local '
|
|
'option.')
|
|
|
|
if not args.install_dir:
|
|
raise ValueError('Please provide --install-dir option.')
|
|
|
|
snapshot_version = args.snapshot_version
|
|
|
|
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
|
|
verbosity = min(args.verbose, 2)
|
|
logging.basicConfig(
|
|
format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
|
|
level=verbose_map[verbosity])
|
|
|
|
install_dir = os.path.expanduser(args.install_dir)
|
|
if os.path.exists(install_dir):
|
|
def remove_dir():
|
|
logging.info('Removing {}'.format(install_dir))
|
|
check_call(['rm', '-rf', install_dir])
|
|
if args.overwrite:
|
|
remove_dir()
|
|
else:
|
|
resp = input('Directory {} already exists. IT WILL BE REMOVED.\n'
|
|
'Are you sure? (yes/no): '.format(install_dir))
|
|
if resp == 'yes':
|
|
remove_dir()
|
|
elif resp == 'no':
|
|
logging.info('Cancelled snapshot install.')
|
|
return
|
|
else:
|
|
raise ValueError('Did not understand: ' + resp)
|
|
check_call(['mkdir', '-p', install_dir])
|
|
|
|
install_artifacts(
|
|
branch=args.branch,
|
|
build=args.build,
|
|
target=args.target,
|
|
local_dir=local,
|
|
symlink=args.symlink,
|
|
install_dir=install_dir)
|
|
gen_bp_files(install_dir, snapshot_version)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|