diff --git a/vndk/tools/header-checker/utils/create_reference_dumps.py b/vndk/tools/header-checker/utils/create_reference_dumps.py index 160bf75db..91019195f 100755 --- a/vndk/tools/header-checker/utils/create_reference_dumps.py +++ b/vndk/tools/header-checker/utils/create_reference_dumps.py @@ -1,22 +1,25 @@ #!/usr/bin/env python3 -import os -import re -import sys -import subprocess import argparse +import collections +import os +import subprocess import time -from utils import (make_libraries, make_tree, find_lib_lsdumps, - get_build_vars_for_product, AOSP_DIR, read_output_content, - copy_reference_dumps, COMPRESSED_SOURCE_ABI_DUMP_EXT, - SOURCE_ABI_DUMP_EXT, SOURCE_ABI_DUMP_EXT_END, SO_EXT, - make_targets) +from utils import ( + AOSP_DIR, COMPRESSED_SOURCE_ABI_DUMP_EXT, SOURCE_ABI_DUMP_EXT_END, SO_EXT, + copy_reference_dumps, find_lib_lsdumps, get_build_vars_for_product, + make_libraries, make_tree) + + +PRODUCTS_DEFAULT = ['aosp_arm_ab', 'aosp_arm', 'aosp_arm64', 'aosp_x86_ab', + 'aosp_x86', 'aosp_x86_64'] + +PREBUILTS_ABI_DUMPS_DEFAULT = os.path.join(AOSP_DIR, 'prebuilts', 'abi-dumps') -PRODUCTS = ['aosp_arm_ab', 'aosp_arm', 'aosp_arm64', 'aosp_x86_ab', 'aosp_x86', - 'aosp_x86_64'] SOONG_DIR = os.path.join(AOSP_DIR, 'out', 'soong', '.intermediates') + class Target(object): def __init__(self, has_2nd, product): extra = '_2ND' if has_2nd else '' @@ -30,29 +33,32 @@ class Target(object): self.arch_variant = build_vars[2] self.cpu_variant = build_vars[3] + def get_lsdump_paths(product, libs): if libs is None: return get_lsdump_paths_from_out(product) return search_for_lsdump_paths(SOONG_DIR, libs) + def get_lsdump_paths_from_out(product): build_vars_to_fetch = ['OUT_DIR', 'TARGET_DEVICE'] build_vars = get_build_vars_for_product(build_vars_to_fetch, product) - lsdump_paths_file = os.path.join(AOSP_DIR, build_vars[0],'target', - 'product', build_vars[1], - 'lsdump_paths.txt') - assert(os.path.exists(lsdump_paths_file) == True) - lsdump_paths = dict() + lsdump_paths_file = os.path.join( + AOSP_DIR, build_vars[0], 'target', 'product', build_vars[1], + 'lsdump_paths.txt') + assert os.path.exists(lsdump_paths_file) + lsdump_paths = collections.defaultdict(list) with open(lsdump_paths_file) as f: for path in f: add_to_path_dict(path.rstrip(), lsdump_paths) return lsdump_paths + def get_lib_arch_str(target): - assert(target.primary_arch != '') + assert target.primary_arch != '' target_arch_variant_str = '' - # if TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant empty - # this is the case for aosp_x86_64_ab and aosp_x86 + # If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant + # empty. This is the case for aosp_x86_64 and aosp_x86_ab. if target.arch_variant != target.arch: target_arch_variant_str = '_' + target.arch_variant return target.arch + target_arch_variant_str @@ -65,23 +71,23 @@ def find_and_copy_lib_lsdumps(target, ref_dump_dir_stem, ref_dump_dir_insertion, target.cpu_variant, lsdump_paths, core_or_vendor_shared_str, libs) - # Copy the contents of the lsdump into it's corresponding - # reference directory. + # Copy the contents of the lsdump into their corresponding reference ABI + # dumps directories. return copy_reference_dumps(arch_lsdump_paths, ref_dump_dir_stem, ref_dump_dir_insertion, get_lib_arch_str(target), compress) -def choose_vndk_version(args_version, platform_vndk_version, - board_vndk_version): - version = args_version + +def choose_vndk_version(version, platform_vndk_version, board_vndk_version): if version is None: - # This logic is to be kept in sync with the references directory logic - # in build/soong/library.go . + # This logic must be in sync with the logic for reference ABI dumps + # directory in `build/soong/cc/library.go`. version = platform_vndk_version - if board_vndk_version != 'current' and board_vndk_version != '': + if board_vndk_version not in ('current', ''): version = board_vndk_version return version + def get_ref_dump_dir_stem(args, vndk_or_ndk, product, chosen_vndk_version): binder_bitness = '64' if get_build_vars_for_product(['BINDER32BIT'], product)[0] == 'true': @@ -92,6 +98,7 @@ def get_ref_dump_dir_stem(args, vndk_or_ndk, product, chosen_vndk_version): return ref_dump_dir_stem + def make_libs_for_product(libs, llndk_mode, product): print('making libs for product:', product) if libs: @@ -99,6 +106,7 @@ def make_libs_for_product(libs, llndk_mode, product): else: make_tree(product) + def find_and_remove_path(root_path, file_name=None): if file_name is not None: root_path = os.path.join(root_path, 'source-based', file_name) @@ -106,21 +114,22 @@ def find_and_remove_path(root_path, file_name=None): print('removing', root_path) subprocess.check_call(remove_cmd_str, shell=True) + def remove_references_for_all_arches_and_variants(args, product, targets, chosen_vndk_version): print('Removing reference dumps...') libs = args.libs for target in targets: - if target.arch == '' or target.arch_variant == '': + if target.arch == '' or target.arch_variant == '': continue + dir_to_remove_vndk = os.path.join( - get_ref_dump_dir_stem(args, 'vndk', product, - chosen_vndk_version), - get_lib_arch_str(target)) + get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version), + get_lib_arch_str(target)) + dir_to_remove_ndk = os.path.join( - get_ref_dump_dir_stem(args, 'ndk', product, - chosen_vndk_version), - get_lib_arch_str(target)) + get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version), + get_lib_arch_str(target)) if libs: for lib in libs: @@ -132,40 +141,44 @@ def remove_references_for_all_arches_and_variants(args, product, targets, find_and_remove_path(dir_to_remove_vndk) find_and_remove_path(dir_to_remove_ndk) -def add_to_path_dict(path, dictionary, libs=[]): + +def add_to_path_dict(path, dictionary, libs=tuple()): name, lsdump_ext = os.path.splitext(path) sofile, so_ext = os.path.splitext(name) libname = os.path.basename(sofile) if lsdump_ext == SOURCE_ABI_DUMP_EXT_END and so_ext == SO_EXT: - if libs and (libname not in libs): + if libs and libname not in libs: return - if libname not in dictionary.keys(): - dictionary[libname] = [path] - else: - dictionary[libname].append(path) + dictionary[libname].append(path) + def search_for_lsdump_paths(soong_dir, libs): - lsdump_paths = dict() - for root, dirs, files in os.walk(soong_dir): + lsdump_paths = collections.defaultdict(list) + for root, _, files in os.walk(soong_dir): for file in files: - add_to_path_dict(os.path.join(root, file), lsdump_paths, libs) + add_to_path_dict(os.path.join(root, file), lsdump_paths, libs) return lsdump_paths + def create_source_abi_reference_dumps(args, product, chosen_vndk_version, lsdump_paths, targets): - ref_dump_dir_stem_vndk =\ + ref_dump_dir_stem_vndk = \ get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version) - ref_dump_dir_stem_ndk =\ + ref_dump_dir_stem_ndk = \ get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version) ref_dump_dir_insertion = 'source-based' + num_libs_copied = 0 + for target in targets: - if target.arch == '' or target.arch_variant == '': + if target.arch == '' or target.arch_variant == '': continue + print('Creating dumps for target_arch:', target.arch, 'and variant ', target.arch_variant) - assert(target.primary_arch != '') + assert target.primary_arch != '' + num_libs_copied += find_and_copy_lib_lsdumps( target, ref_dump_dir_stem_vndk, ref_dump_dir_insertion, '_vendor_shared', args.libs, lsdump_paths, args.compress) @@ -177,51 +190,77 @@ def create_source_abi_reference_dumps(args, product, return num_libs_copied -def main(): - # Parse command line options. - assert 'ANDROID_BUILD_TOP' in os.environ - start = time.time() - parser = argparse.ArgumentParser() - parser.add_argument('--version', help='VNDK version') - parser.add_argument('--no-make-lib', help='no m -j lib.vendor while \ - creating reference', default=False, action='store_true') - parser.add_argument('--llndk', help='The libs specified by -l are llndk', - default=False, action='store_true') - parser.add_argument('-libs', help='libs to create references for', - action='append') - parser.add_argument('-products', help='products to create references for', - action='append') - parser.add_argument('-ref-dump-dir', - help='directory to copy reference abi dumps into', - default=os.path.join(AOSP_DIR,'prebuilts/abi-dumps')) - parser.add_argument('--compress', action='store_true', - help='compress reference dump with gzip') - args = parser.parse_args() +def create_source_abi_reference_dumps_for_all_products(args): + """Create reference ABI dumps for all specified products.""" + + platform_vndk_version, board_vndk_version = get_build_vars_for_product( + ['PLATFORM_VNDK_VERSION', 'BOARD_VNDK_VERSION']) + chosen_vndk_version = choose_vndk_version( + args.version, platform_vndk_version, board_vndk_version) + num_processed = 0 - # Remove reference dumps specified by libs / all of them if none specified, - # so that we may build those libraries succesfully. - vndk_versions = get_build_vars_for_product(['PLATFORM_VNDK_VERSION', - 'BOARD_VNDK_VERSION']) - platform_vndk_version = vndk_versions[0] - board_vndk_version = vndk_versions[1] - chosen_vndk_version = \ - choose_vndk_version(args.version, platform_vndk_version, - board_vndk_version) - if args.products is None: - args.products = PRODUCTS + for product in args.products: targets = [Target(True, product), Target(False, product)] - remove_references_for_all_arches_and_variants(args, product, targets, - chosen_vndk_version) - # make all the libs specified / the 'vndk' target if none specified - if (args.no_make_lib == False): + + # Remove reference ABI dumps specified in `args.libs` (or remove all of + # them if none of them are specified) so that we may build these + # libraries successfully. + remove_references_for_all_arches_and_variants( + args, product, targets, chosen_vndk_version) + + if not args.no_make_lib: + # Build all the specified libs (or build the 'vndk' target if none + # of them are specified.) make_libs_for_product(args.libs, args.llndk, product) + lsdump_paths = get_lsdump_paths(product, args.libs) num_processed += create_source_abi_reference_dumps( args, product, chosen_vndk_version, lsdump_paths, targets) - print() + + return num_processed + + +def _parse_args(): + """Parse the command line arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('--version', help='VNDK version') + parser.add_argument('--no-make-lib', action='store_true', + help='no m -j lib.vendor while creating reference') + parser.add_argument('--llndk', action='store_true', + help='The libs specified by -l are llndk') + parser.add_argument('-libs', action='append', + help='libs to create references for') + parser.add_argument('-products', action='append', + help='products to create references for') + parser.add_argument('--compress', action='store_true', + help='compress reference dump with gzip') + parser.add_argument('-ref-dump-dir', + help='directory to copy reference abi dumps into', + default=PREBUILTS_ABI_DUMPS_DEFAULT) + + args = parser.parse_args() + + if args.products is None: + # If `args.products` is unspecified, generate reference ABI dumps for + # all products. + args.products = PRODUCTS_DEFAULT + + return args + + +def main(): + args = _parse_args() + + start = time.time() + num_processed = create_source_abi_reference_dumps_for_all_products(args) end = time.time() + + print() print('msg: Processed', num_processed, 'libraries in ', (end - start) / 60, ' minutes') + + if __name__ == '__main__': main() diff --git a/vndk/tools/header-checker/utils/utils.py b/vndk/tools/header-checker/utils/utils.py index 48db85b44..8f2205381 100644 --- a/vndk/tools/header-checker/utils/utils.py +++ b/vndk/tools/header-checker/utils/utils.py @@ -1,14 +1,20 @@ #!/usr/bin/env python3 -import tempfile +import gzip import os import subprocess -import gzip -import shutil -import time +import sys +import tempfile + SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) -AOSP_DIR = os.getenv('ANDROID_BUILD_TOP') + +try: + AOSP_DIR = os.environ['ANDROID_BUILD_TOP'] +except KeyError: + print('error: ANDROID_BUILD_TOP environment variable is not set.', + file=sys.stderr) + sys.exit(1) BUILTIN_HEADERS_DIR = ( os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'), @@ -33,6 +39,7 @@ DEFAULT_FORMAT = 'ProtobufTextFormat' TARGET_ARCHS = ['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64'] + def get_reference_dump_dir(reference_dump_dir_stem, reference_dump_dir_insertion, lib_arch): reference_dump_dir = os.path.join(reference_dump_dir_stem, lib_arch) @@ -52,6 +59,7 @@ def copy_reference_dumps(lib_paths, reference_dir_stem, num_created += 1 return num_created + def copy_reference_dump(lib_path, reference_dump_dir, compress): reference_dump_path = os.path.join( reference_dump_dir, os.path.basename(lib_path)) @@ -68,6 +76,7 @@ def copy_reference_dump(lib_path, reference_dump_dir, compress): print('Created abi dump at ', reference_dump_path) return reference_dump_path + def copy_reference_dump_content(lib_name, output_content, reference_dump_dir_stem, reference_dump_dir_insertion, lib_arch): @@ -83,25 +92,28 @@ def copy_reference_dump_content(lib_name, output_content, print('Created abi dump at ', reference_dump_path) return reference_dump_path + def read_output_content(output_path, replace_str): with open(output_path, 'r') as f: return f.read().replace(replace_str, '') -def run_header_abi_dumper(input_path, remove_absolute_paths, cflags=[], - export_include_dirs = EXPORTED_HEADERS_DIR): + +def run_header_abi_dumper(input_path, remove_absolute_paths, cflags=tuple(), + export_include_dirs=EXPORTED_HEADERS_DIR): with tempfile.TemporaryDirectory() as tmp: output_path = os.path.join(tmp, os.path.basename(input_path)) + '.dump' run_header_abi_dumper_on_file(input_path, output_path, export_include_dirs, cflags) + if remove_absolute_paths: + return read_output_content(output_path, AOSP_DIR) with open(output_path, 'r') as f: - if remove_absolute_paths: - return read_output_content(output_path, AOSP_DIR) - else: - return f.read() + return f.read() + def run_header_abi_dumper_on_file(input_path, output_path, - export_include_dirs=[], cflags=[], flags=[]): - input_name, input_ext = os.path.splitext(input_path) + export_include_dirs=tuple(), cflags=tuple(), + flags=tuple()): + input_ext = os.path.splitext(input_path)[1] cmd = ['header-abi-dumper', '-o', output_path, input_path,] for dir in export_include_dirs: cmd += ['-I', dir] @@ -110,7 +122,7 @@ def run_header_abi_dumper_on_file(input_path, output_path, cmd += ['-output-format', DEFAULT_FORMAT] cmd += ['--'] cmd += cflags - if input_ext == '.cpp' or input_ext == '.cc' or input_ext == '.h': + if input_ext in ('.cpp', '.cc', '.h'): cmd += DEFAULT_CPPFLAGS else: cmd += DEFAULT_CFLAGS @@ -122,21 +134,21 @@ def run_header_abi_dumper_on_file(input_path, output_path, cmd += ['-I', dir] subprocess.check_call(cmd) + def run_header_abi_linker(output_path, inputs, version_script, api, arch, - flags=[]): + flags=tuple()): """Link inputs, taking version_script into account""" - with tempfile.TemporaryDirectory() as tmp: - cmd = ['header-abi-linker', '-o', output_path, '-v', version_script, - '-api', api, '-arch', arch] - cmd += flags - if '-input-format' not in flags: - cmd += ['-input-format', DEFAULT_FORMAT] - if '-output-format' not in flags: - cmd += ['-output-format', DEFAULT_FORMAT] - cmd += inputs - subprocess.check_call(cmd) - with open(output_path, 'r') as f: - return read_output_content(output_path, AOSP_DIR) + cmd = ['header-abi-linker', '-o', output_path, '-v', version_script, + '-api', api, '-arch', arch] + cmd += flags + if '-input-format' not in flags: + cmd += ['-input-format', DEFAULT_FORMAT] + if '-output-format' not in flags: + cmd += ['-output-format', DEFAULT_FORMAT] + cmd += inputs + subprocess.check_call(cmd) + return read_output_content(output_path, AOSP_DIR) + def make_tree(product): # To aid creation of reference dumps. @@ -144,6 +156,7 @@ def make_tree(product): 'vndk', 'findlsdumps', 'TARGET_PRODUCT=' + product] subprocess.check_call(make_cmd, cwd=AOSP_DIR) + def make_targets(targets, product): make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j'] for target in targets: @@ -152,6 +165,7 @@ def make_targets(targets, product): subprocess.check_call(make_cmd, cwd=AOSP_DIR, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + def make_libraries(libs, product, llndk_mode): # To aid creation of reference dumps. Makes lib.vendor for the current # configuration. @@ -161,6 +175,7 @@ def make_libraries(libs, product, llndk_mode): lib_targets.append(lib) make_targets(lib_targets, product) + def find_lib_lsdumps(target_arch, target_arch_variant, target_cpu_variant, lsdump_paths, core_or_vendor_shared_str, libs): @@ -179,18 +194,19 @@ def find_lib_lsdumps(target_arch, target_arch_variant, target_dir = 'android_' + target_arch + arch_variant +\ cpu_variant + core_or_vendor_shared_str - for key, value in lsdump_paths.items(): - if libs and key not in libs: - continue - for path in lsdump_paths[key]: - if target_dir in path: - arch_lsdump_paths.append(os.path.join(AOSP_DIR, path.strip())) + for key in lsdump_paths: + if libs and key not in libs: + continue + for path in lsdump_paths[key]: + if target_dir in path: + arch_lsdump_paths.append(os.path.join(AOSP_DIR, path.strip())) return arch_lsdump_paths + def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name, - flags=[]): + flags=tuple()): abi_diff_cmd = ['header-abi-diff', '-new', new_test_dump_path, '-old', - old_test_dump_path, '-arch', arch, '-lib', lib_name] + old_test_dump_path, '-arch', arch, '-lib', lib_name] with tempfile.TemporaryDirectory() as tmp: output_name = os.path.join(tmp, lib_name) + '.abidiff' abi_diff_cmd += ['-o', output_name] @@ -208,24 +224,27 @@ def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name, def get_build_vars_for_product(names, product=None): - build_vars_list = [] """ Get build system variable for the launched target.""" + if product is None and 'ANDROID_PRODUCT_OUT' not in os.environ: return None + cmd = '' if product is not None: - cmd += 'source build/envsetup.sh>/dev/null && lunch>/dev/null ' + product + '&&' - cmd += ' build/soong/soong_ui.bash --dumpvars-mode -vars \"' - for name in names: - cmd += name + ' ' + cmd += 'source build/envsetup.sh > /dev/null && ' + cmd += 'lunch ' + product + ' > /dev/null && ' + cmd += 'build/soong/soong_ui.bash --dumpvars-mode -vars \"' + cmd += ' '.join(names) cmd += '\"' proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, cwd=AOSP_DIR, shell=True) - out, err = proc.communicate() + out, _ = proc.communicate() - build_vars = out.decode('utf-8').strip().split('\n') + build_vars = out.decode('utf-8').strip().splitlines() + + build_vars_list = [] for build_var in build_vars: - key, _, value = build_var.partition('=') - build_vars_list.append(value.replace('\'', '')) + value = build_var.partition('=')[2] + build_vars_list.append(value.replace('\'', '')) return build_vars_list