diff --git a/vndk/tools/header-checker/utils/create_reference_dumps.py b/vndk/tools/header-checker/utils/create_reference_dumps.py index 6be3d19b5..d82a2cb86 100755 --- a/vndk/tools/header-checker/utils/create_reference_dumps.py +++ b/vndk/tools/header-checker/utils/create_reference_dumps.py @@ -3,46 +3,164 @@ import os import re import sys +import subprocess import argparse +import time -from utils import (make_library, find_lib_lsdumps, get_build_var, AOSP_DIR, - read_output_content, copy_reference_dumps) +from utils import (make_libraries, make_tree, find_lib_lsdumps, + get_build_vars_for_product, AOSP_DIR, read_output_content, + copy_reference_dumps, COMPRESSED_SOURCE_ABI_DUMP_EXT, + make_targets) + +PRODUCTS = ['aosp_arm_ab', 'aosp_arm64_ab', 'aosp_x86_ab', 'aosp_x86_64_ab'] +FIND_LSDUMPS_TARGET = 'findlsdumps' class Target(object): - def __init__(self, has_2nd): + def __init__(self, has_2nd, product): extra = '_2ND' if has_2nd else '' - self.arch = get_build_var('TARGET{}_ARCH'.format(extra)) - self.arch_variant = get_build_var('TARGET{}_ARCH_VARIANT'.format(extra)) - self.cpu_variant = \ - get_build_var('TARGET{}_CPU_VARIANT'.format(extra)) + build_vars_to_fetch = ['TARGET_ARCH', + 'TARGET{}_ARCH'.format(extra), + 'TARGET{}_ARCH_VARIANT'.format(extra), + 'TARGET{}_CPU_VARIANT'.format(extra)] + build_vars = get_build_vars_for_product(build_vars_to_fetch, product) + self.primary_arch = build_vars[0] + self.arch = build_vars[1] + self.arch_variant = build_vars[2] + self.cpu_variant = build_vars[3] -def create_source_abi_reference_dumps(soong_dir, args): - ref_dump_dir_stem = os.path.join(args.ref_dump_dir, args.version) +def get_lsdump_paths_file(product): + build_vars_to_fetch = ['OUT_DIR', 'TARGET_DEVICE'] + build_vars = get_build_vars_for_product(build_vars_to_fetch, product) + lsdump_paths_file = os.path.join(AOSP_DIR, build_vars[0],'target', + 'product', build_vars[1], + 'lsdump_paths.txt') + if os.path.exists(lsdump_paths_file) == False: + make_targets([FIND_LSDUMPS_TARGET], product) + return lsdump_paths_file + +def get_lsdump_paths_from_out(file_path): + with open(file_path) as f: + return f.read().split(' ') + +def find_and_copy_lib_lsdumps(target, soong_dir, ref_dump_dir_stem, + ref_dump_dir_insertion, + core_or_vendor_shared_str, + libs, lsdump_paths): + assert(target.primary_arch != '') + arch_lsdump_paths = find_lib_lsdumps(target.arch, target.arch_variant, + target.cpu_variant, lsdump_paths, + core_or_vendor_shared_str, + libs) + # Copy the contents of the lsdump into it's corresponding + # reference directory. + return copy_reference_dumps(arch_lsdump_paths, ref_dump_dir_stem, + ref_dump_dir_insertion, + target.arch + '_' + target.arch_variant) + +def get_ref_dump_dir_stem(args, vndk_or_ndk, product, platform_vndk_version): + version = args.version + if version is None: + version = platform_vndk_version + if version != '' and version[0].isdigit() == False : + version = 'current' + primary_arch =\ + get_build_vars_for_product(['TARGET_ARCH'], product)[0] + ref_dump_dir_stem = os.path.join(args.ref_dump_dir, vndk_or_ndk) + ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, version) + ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, primary_arch) + + return ref_dump_dir_stem + +def make_libs_for_all_arches_and_variants(libs): + for product in PRODUCTS: + get_lsdump_paths_file(product) + if libs: + print('making libs for product:', product) + make_libraries(libs, product) + else: + print('making all libs for product: ', product) + make_tree(product) + +def find_and_remove_path(root_path, file_name=None): + if file_name is not None: + print('removing', file_name, 'from root', root_path) + remove_cmd_str = 'find ' + root_path + ' -name ' + file_name +\ + ' -exec rm -rf {} \;' + subprocess.check_call(remove_cmd_str, cwd=AOSP_DIR, shell=True) + else: + remove_cmd_str = 'rm -rf ' + root_path + subprocess.check_call(remove_cmd_str, cwd=AOSP_DIR, shell=True) + +def remove_references_for_all_arches_and_variants(args): + print('Removing reference dumps...') + libs = args.libs + for product in PRODUCTS: + if libs: + for lib in libs: + find_and_remove_path(args.ref_dump_dir, + lib + COMPRESSED_SOURCE_ABI_DUMP_EXT) + else: + find_and_remove_path(os.path.join(args.ref_dump_dir, 'ndk')) + find_and_remove_path(os.path.join(args.ref_dump_dir, 'vndk')) + + +def create_source_abi_reference_dumps(soong_dir, args, product, + platform_vndk_version): + ref_dump_dir_stem_vndk =\ + get_ref_dump_dir_stem(args, 'vndk', product, platform_vndk_version) + ref_dump_dir_stem_ndk =\ + get_ref_dump_dir_stem(args, 'ndk', product, platform_vndk_version) ref_dump_dir_insertion = 'source-based' num_libs_copied = 0 - for target in [Target(True), Target(False)]: - arch_lsdump_paths = find_lib_lsdumps(target.arch, target.arch_variant, - target.cpu_variant, soong_dir) - # Copy the contents of the lsdump into it's corresponding - # reference directory. - num_libs_copied += copy_reference_dumps(arch_lsdump_paths, - ref_dump_dir_stem, - ref_dump_dir_insertion, - target.arch) + lsdump_paths_file = get_lsdump_paths_file(product) + lsdump_paths = get_lsdump_paths_from_out(lsdump_paths_file) + for target in [Target(True, product), Target(False, product)]: + if target.arch == '' or target.arch_variant == '': + continue + print('Creating dumps for target_arch:', target.arch, 'and variant ', + target.arch_variant) + assert(target.primary_arch != '') + num_libs_copied += find_and_copy_lib_lsdumps( + target, soong_dir, ref_dump_dir_stem_vndk, ref_dump_dir_insertion, + '_vendor_shared/', args.libs, lsdump_paths) + + num_libs_copied += find_and_copy_lib_lsdumps( + target, soong_dir, ref_dump_dir_stem_ndk, ref_dump_dir_insertion, + '_core_shared/', args.libs, lsdump_paths) + return num_libs_copied def main(): # Parse command line options. + assert 'ANDROID_BUILD_TOP' in os.environ + start = time.time() parser = argparse.ArgumentParser() parser.add_argument('--version', help='VNDK version') + parser.add_argument('--no-make-lib', help='no m -j lib.vendor while \ + creating reference', default=False, action='store_true') + parser.add_argument('-libs', help='libs to create references for', + action='append') parser.add_argument('-ref-dump-dir', - help='directory to copy reference abi dumps into') + help='directory to copy reference abi dumps into', + default=os.path.join(AOSP_DIR,'prebuilts/abi-dumps')) args = parser.parse_args() num_processed = 0 soong_dir = os.path.join(AOSP_DIR, 'out', 'soong', '.intermediates') - num_processed += create_source_abi_reference_dumps(soong_dir, args) + # Remove reference dumps specified by libs / all of them if none specified, + # so that we may build those libraries succesfully. + remove_references_for_all_arches_and_variants(args) + # make all the libs specified / the entire vndk_package if none specified + if (args.no_make_lib == False): + make_libs_for_all_arches_and_variants(args.libs) + + platform_vndk_version =\ + get_build_vars_for_product(['PLATFORM_VNDK_VERSION'])[0] + for product in PRODUCTS: + num_processed += create_source_abi_reference_dumps( + soong_dir, args, product, platform_vndk_version) print() - print('msg: Processed', num_processed, 'libraries') + end = time.time() + print('msg: Processed', num_processed, 'libraries in ', (end - start) / 60) if __name__ == '__main__': main() diff --git a/vndk/tools/header-checker/utils/utils.py b/vndk/tools/header-checker/utils/utils.py index 1b8cfdfe6..4ae28634f 100644 --- a/vndk/tools/header-checker/utils/utils.py +++ b/vndk/tools/header-checker/utils/utils.py @@ -5,9 +5,10 @@ import os import subprocess import gzip import shutil +import time SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) -AOSP_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, *['..'] * 5)) +AOSP_DIR = os.getenv('ANDROID_BUILD_TOP') BUILTIN_HEADERS_DIR = ( os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'), @@ -23,6 +24,8 @@ EXPORTED_HEADERS_DIR = ( SO_EXT = '.so' SOURCE_ABI_DUMP_EXT_END = '.lsdump' SOURCE_ABI_DUMP_EXT = SO_EXT + SOURCE_ABI_DUMP_EXT_END +COMPRESSED_SOURCE_ABI_DUMP_EXT = SOURCE_ABI_DUMP_EXT + '.gz' +VENDOR_SUFFIX = '.vendor' DEFAULT_CPPFLAGS = ['-x', 'c++', '-std=c++11'] DEFAULT_CFLAGS = ['-std=gnu99'] @@ -119,19 +122,37 @@ def run_header_abi_linker(output_path, inputs, version_script, api, arch): with open(output_path, 'r') as f: return read_output_content(output_path, AOSP_DIR) -def make_library(lib_name): - # Create reference dumps for integration tests - make_cmd = ['make', '-j', lib_name] +def make_tree(product): + # To aid creation of reference dumps. + make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j', + 'vndk_package', 'TARGET_PRODUCT=' + product] subprocess.check_call(make_cmd, cwd=AOSP_DIR) +def make_targets(targets, product): + make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j'] + for target in targets: + make_cmd.append(target) + make_cmd.append('TARGET_PRODUCT=' + product) + subprocess.check_call(make_cmd, cwd=AOSP_DIR, stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT) + +def make_libraries(libs, product): + # To aid creation of reference dumps. Makes lib.vendor for the current + # configuration. + lib_targets = [] + for lib in libs: + lib_targets.append(lib + VENDOR_SUFFIX) + make_targets(lib_targets, product) + def find_lib_lsdumps(target_arch, target_arch_variant, - target_cpu_variant, soong_dir): + target_cpu_variant, lsdump_paths, + core_or_vendor_shared_str, libs): """ Find the lsdump corresponding to lib_name for the given arch parameters if it exists""" assert 'ANDROID_PRODUCT_OUT' in os.environ cpu_variant = '_' + target_cpu_variant arch_variant = '_' + target_arch_variant - lsdump_paths = [] + arch_lsdump_paths = [] if target_cpu_variant == 'generic' or target_cpu_variant is None or\ target_cpu_variant == '': cpu_variant = '' @@ -140,16 +161,16 @@ def find_lib_lsdumps(target_arch, target_arch_variant, arch_variant = '' target_dir = 'android_' + target_arch + arch_variant +\ - cpu_variant + '_vendor_shared' - for base, dirnames, filenames in os.walk(soong_dir): - for filename in filenames: - name, ext = os.path.splitext(filename) - sofile, soext = os.path.splitext(name) - if ext == SOURCE_ABI_DUMP_EXT_END and soext == SO_EXT : - path = os.path.join(base, filename) - if target_dir in os.path.dirname(path): - lsdump_paths.append(path) - return lsdump_paths + cpu_variant + core_or_vendor_shared_str + for path in lsdump_paths: + filename = os.path.basename(path) + name, _ = os.path.splitext(filename) + sofile, _ = os.path.splitext(name) + if target_dir in path: + if libs and sofile not in libs: + continue + arch_lsdump_paths.append(os.path.join(AOSP_DIR, path.strip())) + return arch_lsdump_paths def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name, flags=[]): @@ -167,14 +188,25 @@ def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name, return 0 -def get_build_var(name): - """Get build system variable for the launched target.""" - if 'ANDROID_PRODUCT_OUT' not in os.environ: +def get_build_vars_for_product(names, product=None): + build_vars_list = [] + """ Get build system variable for the launched target.""" + if product is None and 'ANDROID_PRODUCT_OUT' not in os.environ: return None - - cmd = ['build/soong/soong_ui.bash', '--dumpvar-mode', name] + cmd = '' + if product is not None: + cmd += 'source build/envsetup.sh>/dev/null && lunch>/dev/null ' + product + '&&' + cmd += ' build/soong/soong_ui.bash --dumpvars-mode -vars \"' + for name in names: + cmd += name + ' ' + cmd += '\"' proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, cwd=AOSP_DIR) + stderr=subprocess.DEVNULL, cwd=AOSP_DIR, shell=True) out, err = proc.communicate() - return out.decode('utf-8').strip() + + build_vars = out.decode('utf-8').strip().split('\n') + for build_var in build_vars: + key, _, value = build_var.partition('=') + build_vars_list.append(value.replace('\'', '')) + return build_vars_list