Merge "Choose ABI dump directory according to the tag on the lsdump"

vndk/tools/header-checker/utils/create_reference_dumps.py

@@ -8,9 +8,9 @@ import time
 
 from utils import (
     AOSP_DIR, COMPRESSED_SOURCE_ABI_DUMP_EXT, SOURCE_ABI_DUMP_EXT,
-    SOURCE_ABI_DUMP_EXT_END, SO_EXT, copy_reference_dumps, find_lib_lsdumps,
+    SOURCE_ABI_DUMP_EXT_END, SO_EXT, Target, copy_reference_dump,
-    get_build_vars_for_product, get_module_variant_dir_name, make_libraries,
+    find_lib_lsdumps, get_build_vars_for_product, make_libraries, make_tree,
-    make_tree, read_lsdump_paths)
+    read_lsdump_paths)
 
 
 PRODUCTS_DEFAULT = ['aosp_arm_ab', 'aosp_arm', 'aosp_arm64', 'aosp_x86_ab',
@@ -21,47 +21,6 @@ PREBUILTS_ABI_DUMPS_DEFAULT = os.path.join(AOSP_DIR, 'prebuilts', 'abi-dumps')
 SOONG_DIR = os.path.join(AOSP_DIR, 'out', 'soong', '.intermediates')
 
 
-class Target(object):
-    def __init__(self, has_2nd, product):
-        extra = '_2ND' if has_2nd else ''
-        build_vars_to_fetch = ['TARGET_ARCH',
-                               'TARGET{}_ARCH'.format(extra),
-                               'TARGET{}_ARCH_VARIANT'.format(extra),
-                               'TARGET{}_CPU_VARIANT'.format(extra)]
-        build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
-        self.primary_arch = build_vars[0]
-        self.arch = build_vars[1]
-        self.arch_variant = build_vars[2]
-        self.cpu_variant = build_vars[3]
-
-
-def get_lib_arch_str(target):
-    assert target.primary_arch != ''
-    target_arch_variant_str = ''
-    # If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant
-    # empty. This is the case for aosp_x86_64 and aosp_x86_ab.
-    if target.arch_variant != target.arch:
-        target_arch_variant_str = '_' + target.arch_variant
-    return target.arch + target_arch_variant_str
-
-
-def find_and_copy_lib_lsdumps(target, ref_dump_dir_stem, ref_dump_dir_insertion,
-                              core_or_vendor_shared_str, libs, lsdump_paths,
-                              compress):
-    module_variant_dir_name = get_module_variant_dir_name(
-        target.arch, target.arch_variant, target.cpu_variant,
-        core_or_vendor_shared_str)
-
-    arch_lsdump_paths = find_lib_lsdumps(
-        module_variant_dir_name, lsdump_paths, libs)
-
-    # Copy the contents of the lsdump into their corresponding reference ABI
-    # dumps directories.
-    return copy_reference_dumps(arch_lsdump_paths, ref_dump_dir_stem,
-                                ref_dump_dir_insertion,
-                                get_lib_arch_str(target), compress)
-
-
 def choose_vndk_version(version, platform_vndk_version, board_vndk_version):
     if version is None:
         # This logic must be in sync with the logic for reference ABI dumps
@@ -72,25 +31,20 @@ def choose_vndk_version(version, platform_vndk_version, board_vndk_version):
     return version
 
 
-def get_ref_dump_dir_stem(args, vndk_or_ndk, product, chosen_vndk_version):
+def make_libs_for_product(libs, product, variant, targets):
-    binder_bitness = '64'
-    if get_build_vars_for_product(['BINDER32BIT'], product)[0] == 'true':
-        binder_bitness = '32'
-    ref_dump_dir_stem = os.path.join(args.ref_dump_dir, vndk_or_ndk)
-    ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, chosen_vndk_version)
-    ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, binder_bitness)
-
-    return ref_dump_dir_stem
-
-
-def make_libs_for_product(libs, llndk_mode, product, variant, targets):
     print('making libs for', product + '-' + variant)
     if libs:
-        make_libraries(product, variant, targets, libs, llndk_mode)
+        make_libraries(product, variant, targets, libs)
     else:
         make_tree(product, variant)
 
 
+def get_ref_dump_dir_stem(ref_dump_dir, category, chosen_vndk_version,
+                          binder_bitness, arch_cpu):
+    return os.path.join(ref_dump_dir, category, chosen_vndk_version,
+                        binder_bitness, arch_cpu)
+
+
 def find_and_remove_path(root_path, file_name=None):
     if file_name is not None:
         root_path = os.path.join(root_path, 'source-based', file_name)
@@ -103,57 +57,56 @@ def find_and_remove_path(root_path, file_name=None):
             shutil.rmtree(root_path)
 
 
-def remove_references_for_all_arches_and_variants(args, product, targets,
+def remove_references_for_all_arches_and_variants(ref_dump_dir,
-                                                  chosen_vndk_version):
+                                                  chosen_vndk_version,
-    libs = args.libs
+                                                  binder_bitness, targets,
+                                                  libs):
     for target in targets:
         if target.arch == '' or target.arch_variant == '':
             continue
-
+        for category in ('ndk', 'platform', 'vndk'):
-        dir_to_remove_vndk = os.path.join(
+            dir_to_remove = get_ref_dump_dir_stem(
-            get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version),
+                ref_dump_dir, category, chosen_vndk_version, binder_bitness,
-            get_lib_arch_str(target))
+                target.get_arch_cpu_str())
-
+            if libs:
-        dir_to_remove_ndk = os.path.join(
+                for lib in libs:
-            get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version),
+                    find_and_remove_path(dir_to_remove,
-            get_lib_arch_str(target))
+                                         lib + SOURCE_ABI_DUMP_EXT)
-
+                    find_and_remove_path(dir_to_remove,
-        if libs:
+                                         lib + COMPRESSED_SOURCE_ABI_DUMP_EXT)
-            for lib in libs:
+            else:
-                find_and_remove_path(dir_to_remove_vndk,
+                find_and_remove_path(dir_to_remove)
-                                     lib + SOURCE_ABI_DUMP_EXT)
-                find_and_remove_path(dir_to_remove_vndk,
-                                     lib + COMPRESSED_SOURCE_ABI_DUMP_EXT)
-                find_and_remove_path(dir_to_remove_ndk,
-                                     lib + SOURCE_ABI_DUMP_EXT)
-                find_and_remove_path(dir_to_remove_ndk,
-                                     lib + COMPRESSED_SOURCE_ABI_DUMP_EXT)
-        else:
-            find_and_remove_path(dir_to_remove_vndk)
-            find_and_remove_path(dir_to_remove_ndk)
 
 
-def add_to_path_dict(path, dictionary, libs=tuple()):
+def tag_to_dir_name(tag):
-    name, lsdump_ext = os.path.splitext(path)
+    if tag == 'NDK':
-    sofile, so_ext = os.path.splitext(name)
+        return 'ndk'
-    libname = os.path.basename(sofile)
+    if tag == 'PLATFORM':
-    if lsdump_ext == SOURCE_ABI_DUMP_EXT_END and so_ext == SO_EXT:
+        return 'platform'
-        if libs and libname not in libs:
+    if tag.startswith('VNDK') or tag == 'LLNDK':
-            return
+        return 'vndk'
-        dictionary[libname].append(path)
+    raise ValueError(tag + 'is not a known tag.')
 
 
-def create_source_abi_reference_dumps(args, product,
+def find_and_copy_lib_lsdumps(ref_dump_dir, chosen_vndk_version,
-                                      chosen_vndk_version, lsdump_paths,
+                              binder_bitness, target, libs, lsdump_paths,
-                                      targets):
+                              compress):
-    ref_dump_dir_stem_vndk = \
+    arch_lsdump_paths = find_lib_lsdumps(lsdump_paths, libs, target)
-        get_ref_dump_dir_stem(args, 'vndk', product, chosen_vndk_version)
-    ref_dump_dir_stem_ndk = \
-        get_ref_dump_dir_stem(args, 'ndk', product, chosen_vndk_version)
-    ref_dump_dir_insertion = 'source-based'
 
+    num_created = 0
+    for tag, path in arch_lsdump_paths:
+        ref_dump_dir_stem = get_ref_dump_dir_stem(
+            ref_dump_dir, tag_to_dir_name(tag), chosen_vndk_version,
+            binder_bitness, target.get_arch_cpu_str())
+        copy_reference_dump(
+            path, os.path.join(ref_dump_dir_stem, 'source-based'), compress)
+        num_created += 1
+    return num_created
+
+
+def create_source_abi_reference_dumps(args, chosen_vndk_version,
+                                      binder_bitness, lsdump_paths, targets):
     num_libs_copied = 0
-
     for target in targets:
         if target.arch == '' or target.arch_variant == '':
             continue
@@ -163,13 +116,8 @@ def create_source_abi_reference_dumps(args, product,
         assert target.primary_arch != ''
 
         num_libs_copied += find_and_copy_lib_lsdumps(
-            target, ref_dump_dir_stem_vndk, ref_dump_dir_insertion,
+            args.ref_dump_dir, chosen_vndk_version, binder_bitness, target,
-            '_vendor_shared', args.libs, lsdump_paths, args.compress)
+            args.libs, lsdump_paths, args.compress)
-
-        num_libs_copied += find_and_copy_lib_lsdumps(
-            target, ref_dump_dir_stem_ndk, ref_dump_dir_insertion,
-            '_core_shared', args.libs, lsdump_paths, args.compress)
-
     return num_libs_copied
 
 
@@ -186,23 +134,29 @@ def create_source_abi_reference_dumps_for_all_products(args):
     for product in args.products:
         targets = [Target(True, product), Target(False, product)]
 
+        if get_build_vars_for_product(['BINDER32BIT'], product)[0] == 'true':
+            binder_bitness = '32'
+        else:
+            binder_bitness = '64'
+
         # Remove reference ABI dumps specified in `args.libs` (or remove all of
         # them if none of them are specified) so that we may build these
         # libraries successfully.
         remove_references_for_all_arches_and_variants(
-            args, product, targets, chosen_vndk_version)
+            args.ref_dump_dir, chosen_vndk_version, binder_bitness, targets,
+            args.libs)
 
         if not args.no_make_lib:
             # Build all the specified libs (or build the 'vndk' target if none
             # of them are specified.)
-            make_libs_for_product(args.libs, args.llndk, product,
+            make_libs_for_product(args.libs, product,
                                   args.build_variant, targets)
 
         lsdump_paths = read_lsdump_paths(product, args.build_variant, targets,
                                          build=False)
 
         num_processed += create_source_abi_reference_dumps(
-            args, product, chosen_vndk_version, lsdump_paths, targets)
+            args, chosen_vndk_version, binder_bitness, lsdump_paths, targets)
 
     return num_processed
 
@@ -215,7 +169,7 @@ def _parse_args():
     parser.add_argument('--no-make-lib', action='store_true',
                         help='no m -j lib.vendor while creating reference')
     parser.add_argument('--llndk', action='store_true',
-                        help='The libs specified by -l are llndk')
+                        help='the flag is deprecated and has no effect')
     parser.add_argument('-libs', action='append',
                         help='libs to create references for')
     parser.add_argument('-products', action='append',

vndk/tools/header-checker/utils/utils.py

@@ -41,24 +41,44 @@ DEFAULT_HEADER_FLAGS = ["-dump-function-declarations"]
 DEFAULT_FORMAT = 'ProtobufTextFormat'
 
 
-def get_reference_dump_dir(reference_dump_dir_stem,
+class Target(object):
-                           reference_dump_dir_insertion, lib_arch):
+    def __init__(self, is_2nd, product):
-    reference_dump_dir = os.path.join(reference_dump_dir_stem, lib_arch)
+        extra = '_2ND' if is_2nd else ''
-    reference_dump_dir = os.path.join(reference_dump_dir,
+        build_vars_to_fetch = ['TARGET_ARCH',
-                                      reference_dump_dir_insertion)
+                               'TARGET{}_ARCH'.format(extra),
-    return reference_dump_dir
+                               'TARGET{}_ARCH_VARIANT'.format(extra),
+                               'TARGET{}_CPU_VARIANT'.format(extra)]
+        build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
+        self.primary_arch = build_vars[0]
+        assert self.primary_arch != ''
+        self.arch = build_vars[1]
+        self.arch_variant = build_vars[2]
+        self.cpu_variant = build_vars[3]
 
+    def get_arch_cpu_str(self):
+        """Return a string that represents the architecture, the architecture
+        variant, and the CPU variant.
 
-def copy_reference_dumps(lib_paths, reference_dir_stem,
+        If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant
-                         reference_dump_dir_insertion, lib_arch, compress):
+        empty. This is the case for aosp_x86_64 and aosp_x86_ab.
-    reference_dump_dir = get_reference_dump_dir(reference_dir_stem,
+        """
-                                                reference_dump_dir_insertion,
+        if not self.arch_variant or self.arch_variant == self.arch:
-                                                lib_arch)
+            arch_variant = ''
-    num_created = 0
+        else:
-    for lib_path in lib_paths:
+            arch_variant = '_' + self.arch_variant
-        copy_reference_dump(lib_path, reference_dump_dir, compress)
+
-        num_created += 1
+        if not self.cpu_variant or self.cpu_variant == 'generic':
-    return num_created
+            cpu_variant = ''
+        else:
+            cpu_variant = '_' + self.cpu_variant
+
+        return self.arch + arch_variant + cpu_variant
+
+    def get_module_variant_dir_name(self, variant_suffix):
+        """Create module variant directory name from the architecture, the
+        architecture variant, the CPU variant, and a variant suffix
+        (e.g. `_core_shared`, `_vendor_shared`, etc)."""
+        return 'android_' + self.get_arch_cpu_str() + variant_suffix
 
 
 def copy_reference_dump(lib_path, reference_dump_dir, compress):
@@ -151,13 +171,14 @@ def make_tree(product, variant):
     return make_targets(product, variant, ['findlsdumps'])
 
 
-def make_libraries(product, variant, targets, libs, llndk_mode):
+def make_libraries(product, variant, targets, libs):
     """Build lsdump files for specific libs."""
     lsdump_paths = read_lsdump_paths(product, variant, targets, build=True)
-    targets = []
+    make_target_paths = []
     for name in libs:
-        targets.extend(lsdump_paths[name].values())
+        make_target_paths.extend(path for tag, path in
-    make_targets(product, variant, targets)
+                                 lsdump_paths[name].values())
+    make_targets(product, variant, make_target_paths)
 
 
 def get_lsdump_paths_file_path(product, variant):
@@ -172,29 +193,36 @@ def _is_sanitizer_variation(variation):
     return variation in {'asan', 'hwasan', 'tsan', 'intOverflow', 'cfi', 'scs'}
 
 
-def _are_sanitizer_variations(variations):
+def _tag_to_variant_suffix(tag):
-    """Check whether these variations are introduced by sanitizers."""
+    """Map a tag to a variant suffix."""
-    if isinstance(variations, str):
+    if tag in ('LLNDK', 'NDK', 'PLATFORM'):
-        variations = [v for v in variations.split('_') if v]
+        return '_core_shared'
-    return all(_is_sanitizer_variation(v) for v in variations)
+    if tag.startswith('VNDK'):
+        return '_vendor_shared'
+    raise ValueError(tag + ' is not a known tag.')
 
 
 def _read_lsdump_paths(lsdump_paths_file_path, targets):
-    """Read lsdump path from lsdump_paths.txt for each libname and variant."""
+    """Read lsdump paths from lsdump_paths.txt for each libname and variant.
+
+    This function returns a dictionary, {lib_name: {arch_cpu: (tag, path)}}.
+    For example,
+    {
+      "libc": {
+        "x86_x86_64": (
+          "NDK",
+          "path/to/libc.so.lsdump"
+        )
+      }
+    }
+    """
     lsdump_paths = collections.defaultdict(dict)
     suffixes = collections.defaultdict(dict)
 
-    prefixes = []
-    prefixes.extend(get_module_variant_dir_name(
-        target.arch, target.arch_variant, target.cpu_variant, '_core_shared')
-        for target in targets)
-    prefixes.extend(get_module_variant_dir_name(
-        target.arch, target.arch_variant, target.cpu_variant, '_vendor_shared')
-        for target in targets)
-
     with open(lsdump_paths_file_path, 'r') as lsdump_paths_file:
         for line in lsdump_paths_file:
-            path = line.strip()
+            tag, path = (x.strip() for x in line.split(':', 1))
+            variant_suffix = _tag_to_variant_suffix(tag)
             if not path:
                 continue
             dirname, filename = os.path.split(path)
@@ -206,16 +234,21 @@ def _read_lsdump_paths(lsdump_paths_file_path, targets):
             variant = os.path.basename(dirname)
             if not variant:
                 continue
-            for prefix in prefixes:
+            for target in targets:
+                prefix = target.get_module_variant_dir_name(variant_suffix)
                 if not variant.startswith(prefix):
                     continue
                 new_suffix = variant[len(prefix):]
-                if not _are_sanitizer_variations(new_suffix):
+                # Skip if the suffix contains APEX variations.
+                new_variations = [x for x in new_suffix.split('_') if x]
+                if new_variations and not all(_is_sanitizer_variation(x)
+                                              for x in new_variations):
                     continue
-                old_suffix = suffixes[libname].get(prefix)
+                arch_cpu = target.get_arch_cpu_str()
+                old_suffix = suffixes[libname].get(arch_cpu)
                 if not old_suffix or new_suffix > old_suffix:
-                    lsdump_paths[libname][prefix] = path
+                    lsdump_paths[libname][arch_cpu] = (tag, path)
-                    suffixes[libname][prefix] = new_suffix
+                    suffixes[libname][arch_cpu] = new_suffix
     return lsdump_paths
 
 
@@ -228,36 +261,26 @@ def read_lsdump_paths(product, variant, targets, build=True):
     return _read_lsdump_paths(lsdump_paths_file_abspath, targets)
 
 
-def get_module_variant_dir_name(arch, arch_variant, cpu_variant,
+def find_lib_lsdumps(lsdump_paths, libs, target):
-                                variant_suffix):
+    """Find the lsdump corresponding to libs for the given target.
-    """Create module variant directory name from the target architecture, the
-    target architecture variant, the target CPU variant, and a variant suffix
-    (e.g. `_core_shared`, `_vendor_shared`, etc)."""
 
-    if not arch_variant or arch_variant == arch:
+    This function returns a list of (tag, absolute_path).
-        arch_variant = ''
+    For example,
-    else:
+    [
-        arch_variant = '_' + arch_variant
+      (
-
+        "NDK",
-    if not cpu_variant or cpu_variant == 'generic':
+        "/path/to/libc.so.lsdump"
-        cpu_variant = ''
+      )
-    else:
+    ]
-        cpu_variant = '_' + cpu_variant
+    """
-
+    arch_cpu = target.get_arch_cpu_str()
-    return 'android_' + arch + arch_variant + cpu_variant + variant_suffix
-
-
-def find_lib_lsdumps(module_variant_dir_name, lsdump_paths, libs):
-    """Find the lsdump corresponding to lib_name for the given module variant
-    if it exists."""
     result = []
-    for lib_name, variations in lsdump_paths.items():
+    if libs:
-        if libs and lib_name not in libs:
+        for lib_name in libs:
-            continue
+            result.append(lsdump_paths[lib_name][arch_cpu])
-        for variation, path in variations.items():
+    else:
-            if variation.startswith(module_variant_dir_name):
+        result.extend(paths[arch_cpu] for paths in lsdump_paths.values())
-                result.append(os.path.join(AOSP_DIR, path.strip()))
+    return [(tag, os.path.join(AOSP_DIR, path)) for tag, path in result]
-    return result
 
 
 def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,