Shenanigans to make create_reference_dumps.py more usable.

- pick up lsdump paths from
  $(OUT_DIR)/lsdump_paths/$(TARGET_PRODUCT)/paths.txt instead of
  searching through out/soong/.intermediates.

- llndk lsdumps also get picked up and copied to ref dump dir root now

- creation of references are more fine grained to take into account
  target arch variant and cpu variant as well.

- references for vndk libs and llndk libs for all target archs and
  variants can be generated with one invocation of the script now.

Test: utils/create_reference_dumps.py -l libjpeg -l libc++ -l libc

Test: utils/create_reference_dumps.py -l libjpeg -l libc++ -l libc
      --no-make-lib

Test: utils/create_reference_dumps.py

Change-Id: I9105900b70c62c01f8282e1782cb38a42f4d4356

vndk/tools/header-checker/utils/create_reference_dumps.py

@@ -3,46 +3,164 @@
 import os
 import re
 import sys
+import subprocess
 import argparse
+import time
 
-from utils import (make_library, find_lib_lsdumps, get_build_var, AOSP_DIR,
+from utils import (make_libraries, make_tree, find_lib_lsdumps,
-                   read_output_content, copy_reference_dumps)
+                   get_build_vars_for_product, AOSP_DIR, read_output_content,
+                   copy_reference_dumps, COMPRESSED_SOURCE_ABI_DUMP_EXT,
+                   make_targets)
+
+PRODUCTS = ['aosp_arm_ab', 'aosp_arm64_ab', 'aosp_x86_ab', 'aosp_x86_64_ab']
+FIND_LSDUMPS_TARGET = 'findlsdumps'
 
 class Target(object):
-    def __init__(self, has_2nd):
+    def __init__(self, has_2nd, product):
         extra = '_2ND' if has_2nd else ''
-        self.arch = get_build_var('TARGET{}_ARCH'.format(extra))
+        build_vars_to_fetch = ['TARGET_ARCH',
-        self.arch_variant = get_build_var('TARGET{}_ARCH_VARIANT'.format(extra))
+                               'TARGET{}_ARCH'.format(extra),
-        self.cpu_variant = \
+                               'TARGET{}_ARCH_VARIANT'.format(extra),
-            get_build_var('TARGET{}_CPU_VARIANT'.format(extra))
+                               'TARGET{}_CPU_VARIANT'.format(extra)]
+        build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
+        self.primary_arch = build_vars[0]
+        self.arch = build_vars[1]
+        self.arch_variant = build_vars[2]
+        self.cpu_variant = build_vars[3]
 
-def create_source_abi_reference_dumps(soong_dir, args):
+def get_lsdump_paths_file(product):
-    ref_dump_dir_stem = os.path.join(args.ref_dump_dir, args.version)
+    build_vars_to_fetch = ['OUT_DIR', 'TARGET_DEVICE']
-    ref_dump_dir_insertion = 'source-based'
+    build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
-    num_libs_copied = 0
+    lsdump_paths_file = os.path.join(AOSP_DIR, build_vars[0],'target',
-    for target in [Target(True), Target(False)]:
+                                     'product', build_vars[1],
+                                     'lsdump_paths.txt')
+    if os.path.exists(lsdump_paths_file) == False:
+        make_targets([FIND_LSDUMPS_TARGET], product)
+    return lsdump_paths_file
+
+def get_lsdump_paths_from_out(file_path):
+    with open(file_path) as f:
+        return f.read().split(' ')
+
+def find_and_copy_lib_lsdumps(target, soong_dir, ref_dump_dir_stem,
+                              ref_dump_dir_insertion,
+                              core_or_vendor_shared_str,
+                              libs, lsdump_paths):
+    assert(target.primary_arch != '')
     arch_lsdump_paths = find_lib_lsdumps(target.arch, target.arch_variant,
-                                             target.cpu_variant, soong_dir)
+                                         target.cpu_variant, lsdump_paths,
+                                         core_or_vendor_shared_str,
+                                         libs)
     # Copy the contents of the lsdump into it's corresponding
     # reference  directory.
-        num_libs_copied += copy_reference_dumps(arch_lsdump_paths,
+    return copy_reference_dumps(arch_lsdump_paths, ref_dump_dir_stem,
-                                                ref_dump_dir_stem,
                                 ref_dump_dir_insertion,
-                                                target.arch)
+                                target.arch + '_' + target.arch_variant)
+
+def get_ref_dump_dir_stem(args, vndk_or_ndk, product, platform_vndk_version):
+    version = args.version
+    if version is None:
+      version = platform_vndk_version
+    if version != '' and version[0].isdigit() == False :
+        version = 'current'
+    primary_arch =\
+        get_build_vars_for_product(['TARGET_ARCH'], product)[0]
+    ref_dump_dir_stem = os.path.join(args.ref_dump_dir, vndk_or_ndk)
+    ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, version)
+    ref_dump_dir_stem = os.path.join(ref_dump_dir_stem, primary_arch)
+
+    return ref_dump_dir_stem
+
+def make_libs_for_all_arches_and_variants(libs):
+    for product in PRODUCTS:
+        get_lsdump_paths_file(product)
+        if libs:
+            print('making libs for product:', product)
+            make_libraries(libs, product)
+        else:
+            print('making all libs for product: ', product)
+            make_tree(product)
+
+def find_and_remove_path(root_path, file_name=None):
+    if file_name is not None:
+        print('removing', file_name, 'from root', root_path)
+        remove_cmd_str = 'find ' + root_path + ' -name ' + file_name +\
+            ' -exec rm -rf {} \;'
+        subprocess.check_call(remove_cmd_str, cwd=AOSP_DIR, shell=True)
+    else:
+        remove_cmd_str = 'rm -rf ' + root_path
+        subprocess.check_call(remove_cmd_str, cwd=AOSP_DIR, shell=True)
+
+def remove_references_for_all_arches_and_variants(args):
+    print('Removing reference dumps...')
+    libs = args.libs
+    for product in PRODUCTS:
+        if libs:
+            for lib in libs:
+                find_and_remove_path(args.ref_dump_dir,
+                                     lib + COMPRESSED_SOURCE_ABI_DUMP_EXT)
+        else:
+            find_and_remove_path(os.path.join(args.ref_dump_dir, 'ndk'))
+            find_and_remove_path(os.path.join(args.ref_dump_dir, 'vndk'))
+
+
+def create_source_abi_reference_dumps(soong_dir, args, product,
+                                      platform_vndk_version):
+    ref_dump_dir_stem_vndk =\
+        get_ref_dump_dir_stem(args, 'vndk', product, platform_vndk_version)
+    ref_dump_dir_stem_ndk =\
+        get_ref_dump_dir_stem(args, 'ndk', product, platform_vndk_version)
+    ref_dump_dir_insertion = 'source-based'
+    num_libs_copied = 0
+    lsdump_paths_file = get_lsdump_paths_file(product)
+    lsdump_paths = get_lsdump_paths_from_out(lsdump_paths_file)
+    for target in [Target(True, product), Target(False, product)]:
+        if target.arch ==  '' or target.arch_variant == '':
+            continue
+        print('Creating dumps for target_arch:', target.arch, 'and variant ',
+              target.arch_variant)
+        assert(target.primary_arch != '')
+        num_libs_copied += find_and_copy_lib_lsdumps(
+            target, soong_dir, ref_dump_dir_stem_vndk, ref_dump_dir_insertion,
+            '_vendor_shared/', args.libs, lsdump_paths)
+
+        num_libs_copied += find_and_copy_lib_lsdumps(
+            target, soong_dir, ref_dump_dir_stem_ndk, ref_dump_dir_insertion,
+            '_core_shared/', args.libs, lsdump_paths)
+
     return num_libs_copied
 
 
 def main():
     # Parse command line options.
+    assert 'ANDROID_BUILD_TOP' in os.environ
+    start = time.time()
     parser = argparse.ArgumentParser()
     parser.add_argument('--version', help='VNDK version')
+    parser.add_argument('--no-make-lib', help='no m -j lib.vendor while \
+                        creating reference', default=False, action='store_true')
+    parser.add_argument('-libs', help='libs to create references for',
+                        action='append')
     parser.add_argument('-ref-dump-dir',
-                        help='directory to copy reference abi dumps into')
+                        help='directory to copy reference abi dumps into',
+                        default=os.path.join(AOSP_DIR,'prebuilts/abi-dumps'))
     args = parser.parse_args()
     num_processed = 0
     soong_dir = os.path.join(AOSP_DIR, 'out', 'soong', '.intermediates')
-    num_processed += create_source_abi_reference_dumps(soong_dir, args)
+    # Remove reference dumps specified by libs / all of them if none specified,
+    # so that we may build those libraries succesfully.
+    remove_references_for_all_arches_and_variants(args)
+    # make all the libs specified / the entire vndk_package if none specified
+    if (args.no_make_lib == False):
+        make_libs_for_all_arches_and_variants(args.libs)
+
+    platform_vndk_version =\
+        get_build_vars_for_product(['PLATFORM_VNDK_VERSION'])[0]
+    for product in PRODUCTS:
+        num_processed += create_source_abi_reference_dumps(
+            soong_dir, args, product, platform_vndk_version)
     print()
-    print('msg: Processed', num_processed, 'libraries')
+    end = time.time()
+    print('msg: Processed', num_processed, 'libraries in ', (end - start) / 60)
 if __name__ == '__main__':
     main()

vndk/tools/header-checker/utils/utils.py

@@ -5,9 +5,10 @@ import os
 import subprocess
 import gzip
 import shutil
+import time
 
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
-AOSP_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, *['..'] * 5))
+AOSP_DIR = os.getenv('ANDROID_BUILD_TOP')
 
 BUILTIN_HEADERS_DIR = (
     os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'),
@@ -23,6 +24,8 @@ EXPORTED_HEADERS_DIR = (
 SO_EXT = '.so'
 SOURCE_ABI_DUMP_EXT_END = '.lsdump'
 SOURCE_ABI_DUMP_EXT = SO_EXT + SOURCE_ABI_DUMP_EXT_END
+COMPRESSED_SOURCE_ABI_DUMP_EXT = SOURCE_ABI_DUMP_EXT + '.gz'
+VENDOR_SUFFIX = '.vendor'
 
 DEFAULT_CPPFLAGS = ['-x', 'c++', '-std=c++11']
 DEFAULT_CFLAGS = ['-std=gnu99']
@@ -119,19 +122,37 @@ def run_header_abi_linker(output_path, inputs, version_script, api, arch):
         with open(output_path, 'r') as f:
             return read_output_content(output_path, AOSP_DIR)
 
-def make_library(lib_name):
+def make_tree(product):
-    # Create reference dumps for integration tests
+    # To aid creation of reference dumps.
-    make_cmd = ['make', '-j', lib_name]
+    make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j',
+                'vndk_package', 'TARGET_PRODUCT=' + product]
     subprocess.check_call(make_cmd, cwd=AOSP_DIR)
 
+def make_targets(targets, product):
+    make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j']
+    for target in targets:
+        make_cmd.append(target)
+    make_cmd.append('TARGET_PRODUCT=' + product)
+    subprocess.check_call(make_cmd, cwd=AOSP_DIR, stdout=subprocess.DEVNULL,
+                          stderr=subprocess.STDOUT)
+
+def make_libraries(libs, product):
+    # To aid creation of reference dumps. Makes lib.vendor for the current
+    # configuration.
+    lib_targets = []
+    for lib in libs:
+        lib_targets.append(lib + VENDOR_SUFFIX)
+    make_targets(lib_targets, product)
+
 def find_lib_lsdumps(target_arch, target_arch_variant,
-                     target_cpu_variant, soong_dir):
+                     target_cpu_variant, lsdump_paths,
+                     core_or_vendor_shared_str, libs):
     """ Find the lsdump corresponding to lib_name for the given arch parameters
         if it exists"""
     assert 'ANDROID_PRODUCT_OUT' in os.environ
     cpu_variant = '_' + target_cpu_variant
     arch_variant = '_' + target_arch_variant
-    lsdump_paths = []
+    arch_lsdump_paths = []
     if target_cpu_variant == 'generic' or target_cpu_variant is None or\
         target_cpu_variant == '':
         cpu_variant = ''
@@ -140,16 +161,16 @@ def find_lib_lsdumps(target_arch, target_arch_variant,
         arch_variant = ''
 
     target_dir = 'android_' + target_arch + arch_variant +\
-    cpu_variant + '_vendor_shared'
+        cpu_variant + core_or_vendor_shared_str
-    for base, dirnames, filenames in os.walk(soong_dir):
+    for path in lsdump_paths:
-        for filename in filenames:
+        filename = os.path.basename(path)
-            name, ext = os.path.splitext(filename)
+        name, _ = os.path.splitext(filename)
-            sofile, soext = os.path.splitext(name)
+        sofile, _ = os.path.splitext(name)
-            if ext == SOURCE_ABI_DUMP_EXT_END and soext == SO_EXT :
+        if target_dir in path:
-                path = os.path.join(base, filename)
+            if libs and sofile not in libs:
-                if target_dir in os.path.dirname(path):
+                continue
-                    lsdump_paths.append(path)
+            arch_lsdump_paths.append(os.path.join(AOSP_DIR, path.strip()))
-    return lsdump_paths
+    return arch_lsdump_paths
 
 def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,
                  flags=[]):
@@ -167,14 +188,25 @@ def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,
     return 0
 
 
-def get_build_var(name):
+def get_build_vars_for_product(names, product=None):
-    """Get build system variable for the launched target."""
+    build_vars_list = []
-    if 'ANDROID_PRODUCT_OUT' not in os.environ:
+    """ Get build system variable for the launched target."""
+    if product is None and 'ANDROID_PRODUCT_OUT' not in os.environ:
         return None
-
+    cmd = ''
-    cmd = ['build/soong/soong_ui.bash', '--dumpvar-mode', name]
+    if product is not None:
+        cmd += 'source build/envsetup.sh>/dev/null && lunch>/dev/null ' + product + '&&'
+    cmd += ' build/soong/soong_ui.bash --dumpvars-mode -vars \"'
+    for name in names:
+        cmd += name + ' '
+    cmd += '\"'
 
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE, cwd=AOSP_DIR)
+                            stderr=subprocess.DEVNULL, cwd=AOSP_DIR, shell=True)
     out, err = proc.communicate()
-    return out.decode('utf-8').strip()
+
+    build_vars = out.decode('utf-8').strip().split('\n')
+    for build_var in build_vars:
+      key, _, value = build_var.partition('=')
+      build_vars_list.append(value.replace('\'', ''))
+    return build_vars_list