LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Implement more features to vendor snapshot script"

Browse Source
This commit is contained in:
Treehugger Robot
2020-08-18 00:50:25 +00:00
committed by Gerrit Code Review
parent 77cd06c038 ad01dffacb
commit 4681e0372a
1 changed files with 188 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
vendor_snapshot/update.py
View File

@@ -30,18 +30,23 @@ import json
INDENT = ' ' * 4
def get_notice_path(module_name):
return os.path.join('NOTICE_FILES', module_name+'.txt')
return os.path.join('NOTICE_FILES', module_name + '.txt')
def get_target_arch(json_rel_path):
return json_rel_path.split('/')[0]
def get_arch(json_rel_path):
return json_rel_path.split('/')[1].split('-')[1]
def get_variation(json_rel_path):
return json_rel_path.split('/')[2]
# convert .bp prop dictionary to .bp prop string
def gen_bp_prop(prop, ind):
bp = ''
@@ -54,9 +59,9 @@ def gen_bp_prop(prop, ind):
if len(val) == 0:
continue
bp += ind + key + ": "
bp += ind + key + ': '
if type(val) == bool:
bp += "true,\n" if val else "false,\n"
bp += 'true,\n' if val else 'false,\n'
elif type(val) == str:
bp += '"%s",\n' % val
elif type(val) == list:
@@ -72,6 +77,7 @@ def gen_bp_prop(prop, ind):
raise TypeError('unsupported type %s for gen_bp_prop' % type(val))
return bp
# Remove non-existent dirs from given list. Emits warning for such dirs.
def remove_invalid_dirs(paths, bp_dir, module_name):
ret = []
@@ -79,10 +85,11 @@ def remove_invalid_dirs(paths, bp_dir, module_name):
if os.path.isdir(os.path.join(bp_dir, path)):
ret.append(path)
else:
logging.warning(
'Dir "%s" of module "%s" does not exist' % (path, module_name))
logging.warning('Dir "%s" of module "%s" does not exist', path,
module_name)
return ret
JSON_TO_BP = {
'ModuleName': 'name',
'RelativeInstallPath': 'relative_install_path',
@@ -109,6 +116,7 @@ SANITIZER_VARIANT_PROPS = {
'src',
}
# Converts parsed json dictionary (which is intermediate) to Android.bp prop
# dictionary. This validates paths such as include directories and init_rc
# files while converting.
@@ -131,11 +139,12 @@ def convert_json_to_bp_prop(json_path, bp_dir):
if key in JSON_TO_BP:
ret[JSON_TO_BP[key]] = prop[key]
else:
logging.warning(
'Unknown prop "%s" of module "%s"' % (key, module_name))
logging.warning('Unknown prop "%s" of module "%s"', key,
module_name)
return ret
def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
prop = {
# These three are common for all snapshot modules.
@@ -154,7 +163,7 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
common_prop[k] = arch_props[arch][k]
continue
for k in list(common_prop.keys()):
if not k in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
if k not in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
del common_prop[k]
# Some keys has to be arch_props to prevent 32-bit only modules from being
@@ -198,30 +207,8 @@ def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
bp += '}\n\n'
return bp
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'snapshot_version',
type=int,
help='Vendor snapshot version to install, e.g. "30".')
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase output verbosity, e.g. "-v", "-vv".')
return parser.parse_args()
def main():
"""Program entry point."""
args = get_args()
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
verbosity = min(args.verbose, 2)
logging.basicConfig(
format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
level=verbose_map[verbosity])
install_dir = os.path.join('prebuilts', 'vendor', 'v'+str(args.snapshot_version))
def gen_bp_files(install_dir, snapshot_version):
# props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json
props = dict()
@@ -254,7 +241,7 @@ def main():
target_arch)
module_name = prop['name']
notice_path = 'NOTICE_FILES/' + module_name + ".txt"
notice_path = 'NOTICE_FILES/' + module_name + '.txt'
if os.path.exists(os.path.join(bp_dir, notice_path)):
prop['notice'] = notice_path
@@ -262,7 +249,7 @@ def main():
if 'sanitize' in prop:
sanitizer_type = prop['sanitize']
# module_name is {name}.{sanitizer_type}; trim sanitizer_type
module_name = module_name[:-len(sanitizer_type)-1]
module_name = module_name[:-len(sanitizer_type) - 1]
# Only leave props for the sanitize variant
for k in list(prop.keys()):
if not k in SANITIZER_VARIANT_PROPS:
@@ -282,15 +269,161 @@ def main():
bp_dir = os.path.join(install_dir, target_arch)
for variation in props[target_arch]:
for name in props[target_arch][variation]:
androidbp += gen_bp_module(
variation,
name,
args.snapshot_version,
androidbp += gen_bp_module(variation, name, snapshot_version,
target_arch,
props[target_arch][variation][name],
bp_dir)
with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f:
logging.info('Generating Android.bp to: {}'.format(f.name))
f.write(androidbp)
def check_call(cmd):
logging.debug('Running `{}`'.format(' '.join(cmd)))
subprocess.check_call(cmd)
def fetch_artifact(branch, build, target, pattern, destination):
"""Fetches build artifacts from Android Build server.
Args:
branch: string, branch to pull build artifacts from
build: string, build number to pull build artifacts from
target: string, target name to pull build artifacts from
pattern: string, pattern of build artifact file name
destination: string, destination to pull build artifact to
"""
fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
cmd = [
fetch_artifact_path, '--branch', branch, '--target', target, '--bid',
build, pattern, destination
]
check_call(cmd)
def install_artifacts(branch, build, target, local_dir, install_dir):
"""Installs vendor snapshot build artifacts to {install_dir}/v{version}.
1) Fetch build artifacts from Android Build server or from local_dir
2) Unzip build artifacts
Args:
branch: string or None, branch name of build artifacts
build: string or None, build number of build artifacts
target: string or None, target name of build artifacts
local_dir: string or None, local dir to pull artifacts from
install_dir: string, directory to install vendor snapshot
temp_artifact_dir: string, temp directory to hold build artifacts fetched
from Android Build server. For 'local' option, is set to None.
"""
artifact_pattern = 'vendor-*.zip'
def unzip_artifacts(artifact_dir):
artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
for artifact in artifacts:
logging.info('Unzipping Vendor snapshot: {}'.format(artifact))
check_call(['unzip', '-qn', artifact, '-d', install_dir])
if branch and build and target:
with tempfile.TemporaryDirectory() as tmpdir:
logging.info(
'Fetching {pattern} from {branch} (bid: {build}, target: {target})'
.format(
pattern=artifact_pattern,
branch=branch,
build=build,
target=target))
fetch_artifact(branch, build, target, artifact_pattern, tmpdir)
unzip_artifacts(tmpdir)
elif local_dir:
logging.info('Fetching local VNDK snapshot from {}'.format(local_dir))
unzip_artifacts(local_dir)
else:
raise RuntimeError('Neither local nor remote fetch information given.')
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'snapshot_version',
type=int,
help='Vendor snapshot version to install, e.g. "30".')
parser.add_argument('--branch', help='Branch to pull build from.')
parser.add_argument('--build', help='Build number to pull.')
parser.add_argument('--target', help='Target to pull.')
parser.add_argument(
'--local',
help=('Fetch local vendor snapshot artifacts from specified local '
'directory instead of Android Build server. '
'Example: --local /path/to/local/dir'))
parser.add_argument(
'--install-dir',
help=(
'Base directory to which vendor snapshot artifacts are installed. '
'Example: --install-dir vendor/<company name>/vendor_snapshot/v30'))
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase output verbosity, e.g. "-v", "-vv".')
return parser.parse_args()
def main():
"""Program entry point."""
args = get_args()
local = None
if args.local:
local = os.path.expanduser(args.local)
if local:
if args.build or args.branch or args.target:
raise ValueError(
'When --local option is set, --branch, --build or --target cannot be '
'specified.')
elif not os.path.isdir(local):
raise RuntimeError(
'The specified local directory, {}, does not exist.'.format(
local))
else:
if not (args.build and args.branch and args.target):
raise ValueError(
'Please provide --branch, --build and --target. Or set --local '
'option.')
if not args.install_dir:
raise ValueError('Please provide --install-dir option.')
snapshot_version = args.snapshot_version
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
verbosity = min(args.verbose, 2)
logging.basicConfig(
format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
level=verbose_map[verbosity])
install_dir = os.path.expanduser(args.install_dir)
if os.path.exists(install_dir):
resp = input('Directory {} already exists. IT WILL BE REMOVED.\n'
'Are you sure? (yes/no): '.format(install_dir))
if resp == 'yes':
logging.info('Removing {}'.format(install_dir))
check_call(['rm', '-rf', install_dir])
elif resp == 'no':
logging.info('Cancelled snapshot install.')
return
else:
raise ValueError('Did not understand: ' + resp)
check_call(['mkdir', '-p', install_dir])
install_artifacts(
branch=args.branch,
build=args.build,
target=args.target,
local_dir=local,
install_dir=install_dir)
gen_bp_files(install_dir, snapshot_version)
if __name__ == '__main__':
main()