diff --git a/vndk/tools/header-checker/utils/create_reference_dumps.py b/vndk/tools/header-checker/utils/create_reference_dumps.py index 19a21fb03..b70ea1473 100755 --- a/vndk/tools/header-checker/utils/create_reference_dumps.py +++ b/vndk/tools/header-checker/utils/create_reference_dumps.py @@ -8,9 +8,9 @@ import time from utils import ( AOSP_DIR, COMPRESSED_SOURCE_ABI_DUMP_EXT, SOURCE_ABI_DUMP_EXT, - SOURCE_ABI_DUMP_EXT_END, SO_EXT, copy_reference_dumps, find_lib_lsdumps, - get_build_vars_for_product, get_module_variant_dir_name, make_libraries, - make_tree, read_lsdump_paths) + SOURCE_ABI_DUMP_EXT_END, SO_EXT, Target, copy_reference_dump, + find_lib_lsdumps, get_build_vars_for_product, make_libraries, make_tree, + read_lsdump_paths) PRODUCTS_DEFAULT = ['aosp_arm_ab', 'aosp_arm', 'aosp_arm64', 'aosp_x86_ab', @@ -21,47 +21,6 @@ PREBUILTS_ABI_DUMPS_DEFAULT = os.path.join(AOSP_DIR, 'prebuilts', 'abi-dumps') SOONG_DIR = os.path.join(AOSP_DIR, 'out', 'soong', '.intermediates') -class Target(object): - def __init__(self, has_2nd, product): - extra = '_2ND' if has_2nd else '' - build_vars_to_fetch = ['TARGET_ARCH', - 'TARGET{}_ARCH'.format(extra), - 'TARGET{}_ARCH_VARIANT'.format(extra), - 'TARGET{}_CPU_VARIANT'.format(extra)] - build_vars = get_build_vars_for_product(build_vars_to_fetch, product) - self.primary_arch = build_vars[0] - self.arch = build_vars[1] - self.arch_variant = build_vars[2] - self.cpu_variant = build_vars[3] - - -def get_lib_arch_str(target): - assert target.primary_arch != '' - target_arch_variant_str = '' - # If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant - # empty. This is the case for aosp_x86_64 and aosp_x86_ab. - if target.arch_variant != target.arch: - target_arch_variant_str = '_' + target.arch_variant - return target.arch + target_arch_variant_str - - -def find_and_copy_lib_lsdumps(target, ref_dump_dir_stem, ref_dump_dir_insertion, - core_or_vendor_shared_str, libs, lsdump_paths, - compress): - module_variant_dir_name = get_module_variant_dir_name( - target.arch, target.arch_variant, target.cpu_variant, - core_or_vendor_shared_str) - - arch_lsdump_paths = find_lib_lsdumps( - module_variant_dir_name, lsdump_paths, libs) - - # Copy the contents of the lsdump into their corresponding reference ABI - # dumps directories. - return copy_reference_dumps(arch_lsdump_paths, ref_dump_dir_stem, - ref_dump_dir_insertion, - get_lib_arch_str(target), compress) - - def choose_vndk_version(version, platform_vndk_version, board_vndk_version): if version is None: # This logic must be in sync with the logic for reference ABI dumps @@ -72,25 +31,20 @@ def choose_vndk_version(version, platform_vndk_version, board_vndk_version): return version -def get_ref_dump_dir_stem(args, vndk_or_ndk, product, chosen_vndk_version): - binder_bitness = '64' - if get_build_vars_for_product(['BINDER32BIT'], product)[0] == 'true': - binder_bitness = '32' - ref_dump_dir_stem = os.path.join(args.ref_dump_dir, vndk_or_ndk) - ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, chosen_vndk_version) - ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, binder_bitness) - - return ref_dump_dir_stem - - -def make_libs_for_product(libs, llndk_mode, product, variant, targets): +def make_libs_for_product(libs, product, variant, targets): print('making libs for', product + '-' + variant) if libs: - make_libraries(product, variant, targets, libs, llndk_mode) + make_libraries(product, variant, targets, libs) else: make_tree(product, variant) +def get_ref_dump_dir_stem(ref_dump_dir, category, chosen_vndk_version, + binder_bitness, arch_cpu): + return os.path.join(ref_dump_dir, category, chosen_vndk_version, + binder_bitness, arch_cpu) + + def find_and_remove_path(root_path, file_name=None): if file_name is not None: root_path = os.path.join(root_path, 'source-based', file_name) @@ -103,57 +57,56 @@ def find_and_remove_path(root_path, file_name=None): shutil.rmtree(root_path) -def remove_references_for_all_arches_and_variants(args, product, targets, - chosen_vndk_version): - libs = args.libs +def remove_references_for_all_arches_and_variants(ref_dump_dir, + chosen_vndk_version, + binder_bitness, targets, + libs): for target in targets: if target.arch == '' or target.arch_variant == '': continue - - dir_to_remove_vndk = os.path.join( - get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version), - get_lib_arch_str(target)) - - dir_to_remove_ndk = os.path.join( - get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version), - get_lib_arch_str(target)) - - if libs: - for lib in libs: - find_and_remove_path(dir_to_remove_vndk, - lib + SOURCE_ABI_DUMP_EXT) - find_and_remove_path(dir_to_remove_vndk, - lib + COMPRESSED_SOURCE_ABI_DUMP_EXT) - find_and_remove_path(dir_to_remove_ndk, - lib + SOURCE_ABI_DUMP_EXT) - find_and_remove_path(dir_to_remove_ndk, - lib + COMPRESSED_SOURCE_ABI_DUMP_EXT) - else: - find_and_remove_path(dir_to_remove_vndk) - find_and_remove_path(dir_to_remove_ndk) + for category in ('ndk', 'platform', 'vndk'): + dir_to_remove = get_ref_dump_dir_stem( + ref_dump_dir, category, chosen_vndk_version, binder_bitness, + target.get_arch_cpu_str()) + if libs: + for lib in libs: + find_and_remove_path(dir_to_remove, + lib + SOURCE_ABI_DUMP_EXT) + find_and_remove_path(dir_to_remove, + lib + COMPRESSED_SOURCE_ABI_DUMP_EXT) + else: + find_and_remove_path(dir_to_remove) -def add_to_path_dict(path, dictionary, libs=tuple()): - name, lsdump_ext = os.path.splitext(path) - sofile, so_ext = os.path.splitext(name) - libname = os.path.basename(sofile) - if lsdump_ext == SOURCE_ABI_DUMP_EXT_END and so_ext == SO_EXT: - if libs and libname not in libs: - return - dictionary[libname].append(path) +def tag_to_dir_name(tag): + if tag == 'NDK': + return 'ndk' + if tag == 'PLATFORM': + return 'platform' + if tag.startswith('VNDK') or tag == 'LLNDK': + return 'vndk' + raise ValueError(tag + 'is not a known tag.') -def create_source_abi_reference_dumps(args, product, - chosen_vndk_version, lsdump_paths, - targets): - ref_dump_dir_stem_vndk = \ - get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version) - ref_dump_dir_stem_ndk = \ - get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version) - ref_dump_dir_insertion = 'source-based' +def find_and_copy_lib_lsdumps(ref_dump_dir, chosen_vndk_version, + binder_bitness, target, libs, lsdump_paths, + compress): + arch_lsdump_paths = find_lib_lsdumps(lsdump_paths, libs, target) + num_created = 0 + for tag, path in arch_lsdump_paths: + ref_dump_dir_stem = get_ref_dump_dir_stem( + ref_dump_dir, tag_to_dir_name(tag), chosen_vndk_version, + binder_bitness, target.get_arch_cpu_str()) + copy_reference_dump( + path, os.path.join(ref_dump_dir_stem, 'source-based'), compress) + num_created += 1 + return num_created + + +def create_source_abi_reference_dumps(args, chosen_vndk_version, + binder_bitness, lsdump_paths, targets): num_libs_copied = 0 - for target in targets: if target.arch == '' or target.arch_variant == '': continue @@ -163,13 +116,8 @@ def create_source_abi_reference_dumps(args, product, assert target.primary_arch != '' num_libs_copied += find_and_copy_lib_lsdumps( - target, ref_dump_dir_stem_vndk, ref_dump_dir_insertion, - '_vendor_shared', args.libs, lsdump_paths, args.compress) - - num_libs_copied += find_and_copy_lib_lsdumps( - target, ref_dump_dir_stem_ndk, ref_dump_dir_insertion, - '_core_shared', args.libs, lsdump_paths, args.compress) - + args.ref_dump_dir, chosen_vndk_version, binder_bitness, target, + args.libs, lsdump_paths, args.compress) return num_libs_copied @@ -186,23 +134,29 @@ def create_source_abi_reference_dumps_for_all_products(args): for product in args.products: targets = [Target(True, product), Target(False, product)] + if get_build_vars_for_product(['BINDER32BIT'], product)[0] == 'true': + binder_bitness = '32' + else: + binder_bitness = '64' + # Remove reference ABI dumps specified in `args.libs` (or remove all of # them if none of them are specified) so that we may build these # libraries successfully. remove_references_for_all_arches_and_variants( - args, product, targets, chosen_vndk_version) + args.ref_dump_dir, chosen_vndk_version, binder_bitness, targets, + args.libs) if not args.no_make_lib: # Build all the specified libs (or build the 'vndk' target if none # of them are specified.) - make_libs_for_product(args.libs, args.llndk, product, + make_libs_for_product(args.libs, product, args.build_variant, targets) lsdump_paths = read_lsdump_paths(product, args.build_variant, targets, build=False) num_processed += create_source_abi_reference_dumps( - args, product, chosen_vndk_version, lsdump_paths, targets) + args, chosen_vndk_version, binder_bitness, lsdump_paths, targets) return num_processed @@ -215,7 +169,7 @@ def _parse_args(): parser.add_argument('--no-make-lib', action='store_true', help='no m -j lib.vendor while creating reference') parser.add_argument('--llndk', action='store_true', - help='The libs specified by -l are llndk') + help='the flag is deprecated and has no effect') parser.add_argument('-libs', action='append', help='libs to create references for') parser.add_argument('-products', action='append', diff --git a/vndk/tools/header-checker/utils/utils.py b/vndk/tools/header-checker/utils/utils.py index 741af4cea..9b0df7dec 100644 --- a/vndk/tools/header-checker/utils/utils.py +++ b/vndk/tools/header-checker/utils/utils.py @@ -41,24 +41,44 @@ DEFAULT_HEADER_FLAGS = ["-dump-function-declarations"] DEFAULT_FORMAT = 'ProtobufTextFormat' -def get_reference_dump_dir(reference_dump_dir_stem, - reference_dump_dir_insertion, lib_arch): - reference_dump_dir = os.path.join(reference_dump_dir_stem, lib_arch) - reference_dump_dir = os.path.join(reference_dump_dir, - reference_dump_dir_insertion) - return reference_dump_dir +class Target(object): + def __init__(self, is_2nd, product): + extra = '_2ND' if is_2nd else '' + build_vars_to_fetch = ['TARGET_ARCH', + 'TARGET{}_ARCH'.format(extra), + 'TARGET{}_ARCH_VARIANT'.format(extra), + 'TARGET{}_CPU_VARIANT'.format(extra)] + build_vars = get_build_vars_for_product(build_vars_to_fetch, product) + self.primary_arch = build_vars[0] + assert self.primary_arch != '' + self.arch = build_vars[1] + self.arch_variant = build_vars[2] + self.cpu_variant = build_vars[3] + def get_arch_cpu_str(self): + """Return a string that represents the architecture, the architecture + variant, and the CPU variant. -def copy_reference_dumps(lib_paths, reference_dir_stem, - reference_dump_dir_insertion, lib_arch, compress): - reference_dump_dir = get_reference_dump_dir(reference_dir_stem, - reference_dump_dir_insertion, - lib_arch) - num_created = 0 - for lib_path in lib_paths: - copy_reference_dump(lib_path, reference_dump_dir, compress) - num_created += 1 - return num_created + If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant + empty. This is the case for aosp_x86_64 and aosp_x86_ab. + """ + if not self.arch_variant or self.arch_variant == self.arch: + arch_variant = '' + else: + arch_variant = '_' + self.arch_variant + + if not self.cpu_variant or self.cpu_variant == 'generic': + cpu_variant = '' + else: + cpu_variant = '_' + self.cpu_variant + + return self.arch + arch_variant + cpu_variant + + def get_module_variant_dir_name(self, variant_suffix): + """Create module variant directory name from the architecture, the + architecture variant, the CPU variant, and a variant suffix + (e.g. `_core_shared`, `_vendor_shared`, etc).""" + return 'android_' + self.get_arch_cpu_str() + variant_suffix def copy_reference_dump(lib_path, reference_dump_dir, compress): @@ -151,13 +171,14 @@ def make_tree(product, variant): return make_targets(product, variant, ['findlsdumps']) -def make_libraries(product, variant, targets, libs, llndk_mode): +def make_libraries(product, variant, targets, libs): """Build lsdump files for specific libs.""" lsdump_paths = read_lsdump_paths(product, variant, targets, build=True) - targets = [] + make_target_paths = [] for name in libs: - targets.extend(lsdump_paths[name].values()) - make_targets(product, variant, targets) + make_target_paths.extend(path for tag, path in + lsdump_paths[name].values()) + make_targets(product, variant, make_target_paths) def get_lsdump_paths_file_path(product, variant): @@ -172,29 +193,36 @@ def _is_sanitizer_variation(variation): return variation in {'asan', 'hwasan', 'tsan', 'intOverflow', 'cfi', 'scs'} -def _are_sanitizer_variations(variations): - """Check whether these variations are introduced by sanitizers.""" - if isinstance(variations, str): - variations = [v for v in variations.split('_') if v] - return all(_is_sanitizer_variation(v) for v in variations) +def _tag_to_variant_suffix(tag): + """Map a tag to a variant suffix.""" + if tag in ('LLNDK', 'NDK', 'PLATFORM'): + return '_core_shared' + if tag.startswith('VNDK'): + return '_vendor_shared' + raise ValueError(tag + ' is not a known tag.') def _read_lsdump_paths(lsdump_paths_file_path, targets): - """Read lsdump path from lsdump_paths.txt for each libname and variant.""" + """Read lsdump paths from lsdump_paths.txt for each libname and variant. + + This function returns a dictionary, {lib_name: {arch_cpu: (tag, path)}}. + For example, + { + "libc": { + "x86_x86_64": ( + "NDK", + "path/to/libc.so.lsdump" + ) + } + } + """ lsdump_paths = collections.defaultdict(dict) suffixes = collections.defaultdict(dict) - prefixes = [] - prefixes.extend(get_module_variant_dir_name( - target.arch, target.arch_variant, target.cpu_variant, '_core_shared') - for target in targets) - prefixes.extend(get_module_variant_dir_name( - target.arch, target.arch_variant, target.cpu_variant, '_vendor_shared') - for target in targets) - with open(lsdump_paths_file_path, 'r') as lsdump_paths_file: for line in lsdump_paths_file: - path = line.strip() + tag, path = (x.strip() for x in line.split(':', 1)) + variant_suffix = _tag_to_variant_suffix(tag) if not path: continue dirname, filename = os.path.split(path) @@ -206,16 +234,21 @@ def _read_lsdump_paths(lsdump_paths_file_path, targets): variant = os.path.basename(dirname) if not variant: continue - for prefix in prefixes: + for target in targets: + prefix = target.get_module_variant_dir_name(variant_suffix) if not variant.startswith(prefix): continue new_suffix = variant[len(prefix):] - if not _are_sanitizer_variations(new_suffix): + # Skip if the suffix contains APEX variations. + new_variations = [x for x in new_suffix.split('_') if x] + if new_variations and not all(_is_sanitizer_variation(x) + for x in new_variations): continue - old_suffix = suffixes[libname].get(prefix) + arch_cpu = target.get_arch_cpu_str() + old_suffix = suffixes[libname].get(arch_cpu) if not old_suffix or new_suffix > old_suffix: - lsdump_paths[libname][prefix] = path - suffixes[libname][prefix] = new_suffix + lsdump_paths[libname][arch_cpu] = (tag, path) + suffixes[libname][arch_cpu] = new_suffix return lsdump_paths @@ -228,36 +261,26 @@ def read_lsdump_paths(product, variant, targets, build=True): return _read_lsdump_paths(lsdump_paths_file_abspath, targets) -def get_module_variant_dir_name(arch, arch_variant, cpu_variant, - variant_suffix): - """Create module variant directory name from the target architecture, the - target architecture variant, the target CPU variant, and a variant suffix - (e.g. `_core_shared`, `_vendor_shared`, etc).""" +def find_lib_lsdumps(lsdump_paths, libs, target): + """Find the lsdump corresponding to libs for the given target. - if not arch_variant or arch_variant == arch: - arch_variant = '' - else: - arch_variant = '_' + arch_variant - - if not cpu_variant or cpu_variant == 'generic': - cpu_variant = '' - else: - cpu_variant = '_' + cpu_variant - - return 'android_' + arch + arch_variant + cpu_variant + variant_suffix - - -def find_lib_lsdumps(module_variant_dir_name, lsdump_paths, libs): - """Find the lsdump corresponding to lib_name for the given module variant - if it exists.""" + This function returns a list of (tag, absolute_path). + For example, + [ + ( + "NDK", + "/path/to/libc.so.lsdump" + ) + ] + """ + arch_cpu = target.get_arch_cpu_str() result = [] - for lib_name, variations in lsdump_paths.items(): - if libs and lib_name not in libs: - continue - for variation, path in variations.items(): - if variation.startswith(module_variant_dir_name): - result.append(os.path.join(AOSP_DIR, path.strip())) - return result + if libs: + for lib_name in libs: + result.append(lsdump_paths[lib_name][arch_cpu]) + else: + result.extend(paths[arch_cpu] for paths in lsdump_paths.values()) + return [(tag, os.path.join(AOSP_DIR, path)) for tag, path in result] def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,