Merge changes I960c33e7,I256be1ee,I56cd01ac,Icb498d88,Ic8204f04

* changes:
  vndk-def: Automate the process to update list
  vndk-def: Add special cases for libclang_rt libs
  sourcedr: Extract load_manifest_from_args()
  sourcedr: Move run_tests.sh
  sourcedr: Reorganize ninja.py code

vndk/tools/definition-tool/tools/update_dataset.py

@@ -59,17 +59,35 @@ def load_install_paths(module_info_path):
 
     return (result, name_path_dict)
 
-def main():
-    parser =argparse.ArgumentParser()
+def _is_stale_module(path, installed_paths):
+    if path in installed_paths:
+        return False
+    # libclang_rt.asan-${arch}-android and
+    # libclang_rt.ubsan_standalone-${arch}-android may vary between different
+    # architectures.
+    if posixpath.basename(path).startswith('libclang_rt'):
+        return False
+    return True
+
+def remove_stale_modules(data, installed_paths):
+    result = {}
+    for path, row in data.items():
+        if not _is_stale_module(path, installed_paths):
+            result[path] = row
+    return result
+
+def _parse_args():
+    parser = argparse.ArgumentParser()
     parser.add_argument('tag_file')
     parser.add_argument('-o', '--output', required=True)
     parser.add_argument('--make-vars', required=True,
                         help='out/soong/make_vars-$(TARGET).mk')
     parser.add_argument('--module-info', required=True,
                         help='out/target/product/$(TARGET)/module-info.json')
-    parser.add_argument('--delete-removed-entries', action='store_true',
-                        help='Delete removed shared libs')
-    args = parser.parse_args()
+    return parser.parse_args()
+
+def main():
+    args = _parse_args()
 
     # Load libraries from `out/soong/make_vars-$(TARGET).mk`.
     llndk, vndk_sp, vndk, vndk_private = load_make_vars(args.make_vars)
@@ -88,12 +106,7 @@ def main():
 
     # Delete non-existing libraries.
     installed_paths, name_path_dict = load_install_paths(args.module_info)
-
-    if args.delete_removed_entries:
-        data = { path: row for path, row in data.items()
-                           if path in installed_paths }
-    else:
-        data = { path: row for path, row in data.items() }
+    data = remove_stale_modules(data, installed_paths)
 
     # Reset all /system/${LIB} libraries to FWK-ONLY.
     for path, row in data.items():

vndk/tools/definition-tool/tools/update_dataset_auto.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+# This tool generates `eligible-list-${ver}.csv` and
+# `eligible-list-${ver}-properties.csv`.
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+
+UPDATE_DATASET = os.path.abspath(os.path.join(
+    __file__, '..', 'update_dataset.py'))
+
+LIST_VNDK_MODULE = os.path.abspath(os.path.join(
+    __file__, '..', '..', '..', 'sourcedr', 'sourcedr', 'blueprint',
+    'list_vndk_module.py'))
+
+
+def update_eligible_list(path, make_vars, module_info):
+    dirname, basename = os.path.split(path)
+    tmp_fd, tmp_path = tempfile.mkstemp(prefix=basename + '-', dir=dirname)
+    os.close(tmp_fd)
+
+    cmd = [sys.executable, UPDATE_DATASET]
+    cmd.extend(['--make-vars', make_vars])
+    cmd.extend(['--module-info', module_info])
+    cmd.extend(['-o', tmp_path])
+    cmd.append(path)
+
+    print('command:', ' '.join(cmd))
+
+    subprocess.check_call(cmd)
+    os.rename(tmp_path, path)
+
+
+def update_eligible_list_properties(path, build_top):
+    dirname, basename = os.path.split(path)
+    tmp_fd, tmp_path = tempfile.mkstemp(prefix=basename + '-', dir=dirname)
+    os.close(tmp_fd)
+
+    cmd = [sys.executable, LIST_VNDK_MODULE]
+    cmd.extend(['--exclude', '(?:device/)|(?:vendor/)'])
+    cmd.extend(['-o', tmp_path])
+    cmd.append(os.path.join(build_top, 'Android.bp'))
+
+    print('command:', ' '.join(cmd))
+
+    subprocess.check_call(cmd)
+    os.rename(tmp_path, path)
+
+
+def _get_eligible_list_properties_path(eligible_list_path):
+    root, ext = os.path.splitext(eligible_list_path)
+    return root + '-properties' + ext
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('eligible_list',
+                        help='Path to eligible list to be updated')
+    return parser.parse_args()
+
+
+def main():
+    args = _parse_args()
+
+    # Read Android product environment variables.
+    try:
+        build_top = os.environ['ANDROID_BUILD_TOP']
+        product = os.environ['TARGET_PRODUCT']
+        product_out = os.environ['ANDROID_PRODUCT_OUT']
+    except KeyError as e:
+        print('error: Failed to read the environment variable', e.args[0],
+              file=sys.stderr)
+        print('error: Did you lunch an Android target?', file=sys.stderr)
+        sys.exit(1)
+
+    print('----------------------------------------')
+    print('build_top:', build_top)
+    print('product:', product)
+    print('product_out:', product_out)
+
+    out_dir = os.path.normpath(os.path.join(product_out, '..', '..', '..'))
+    make_vars = os.path.join(out_dir, 'soong', 'make_vars-' + product + '.mk')
+    module_info = os.path.join(product_out, 'module-info.json')
+
+    print('----------------------------------------')
+    print('make_vars:', make_vars)
+    print('module_info:', module_info)
+    print('----------------------------------------')
+
+    # Run the commands to update the files.
+    update_eligible_list(args.eligible_list, make_vars, module_info)
+    update_eligible_list_properties(
+        _get_eligible_list_properties_path(args.eligible_list), build_top)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

vndk/tools/sourcedr/sourcedr/blueprint/tests/run_tests.sh

@@ -16,5 +16,7 @@
 # limitations under the License.
 #
 
+cd "$(dirname "$0")/.."
+
 python3 -m unittest discover "$@"
 python -m unittest discover "$@"

vndk/tools/sourcedr/sourcedr/ninja/list_installed_file_from_source.py

@@ -37,16 +37,6 @@ def _parse_args():
     return parser.parse_args()
 
 
-def _load_manifest_from_args(args):
-    input_file = args.input_file
-
-    if input_file.endswith('.pickle'):
-        with open(input_file, 'rb') as pickle_file:
-            return pickle.load(pickle_file)
-
-    return Parser(args.cwd).parse(input_file, args.encoding, args.ninja_deps)
-
-
 def main():
     args = _parse_args()
 
@@ -65,7 +55,7 @@ def main():
         '|'.join('(?:' + re.escape(posixpath.normpath(path)) + ')'
                  for path in args.source_filter.split(':')))
 
-    manifest = _load_manifest_from_args(args)
+    manifest = ninja.load_manifest_from_args(args)
 
     # Build lookup map
     outs = {}

vndk/tools/sourcedr/sourcedr/ninja/list_ninja_build_dep.py

@@ -33,21 +33,6 @@ def _parse_args():
     return parser.parse_args()
 
 
-def _load_manifest_from_args(args):
-    """Load the ninja file specified in the command line arguments."""
-
-    input_file = args.input_file
-
-    # If the input file name ends with `.pickle`, load it with pickle.load().
-    if input_file.endswith('.pickle'):
-        with open(input_file, 'rb') as pickle_file:
-            return pickle.load(pickle_file)
-
-    # Parse the ninja file
-    ninja_parser = ninja.Parser(args.cwd)
-    return ninja_parser.parse(input_file, args.encoding, args.ninja_deps)
-
-
 def collect_build_targets(graph, target):
     """Collect the transitive build targets."""
 
@@ -78,7 +63,7 @@ def main():
     args = _parse_args()
 
     # Build lookup map
-    manifest = _load_manifest_from_args(args)
+    manifest = ninja.load_manifest_from_args(args)
     graph = {}
     for build in manifest.builds:
         for path in build.explicit_outs:

vndk/tools/sourcedr/sourcedr/ninja/list_source_file.py

@@ -37,21 +37,6 @@ def _parse_args():
     return parser.parse_args()
 
 
-def _load_manifest_from_args(args):
-    """Load the ninja file specified in the command line arguments."""
-
-    input_file = args.input_file
-
-    # If the input file name ends with `.pickle`, load it with pickle.load().
-    if input_file.endswith('.pickle'):
-        with open(input_file, 'rb') as pickle_file:
-            return pickle.load(pickle_file)
-
-    # Parse the ninja file
-    ninja_parser = ninja.Parser(args.cwd)
-    return ninja_parser.parse(input_file, args.encoding, args.ninja_deps)
-
-
 def collect_source_files(graph, start, out_dir_pattern, out_host_dir_pattern):
     """Collect the transitive dependencies of a target."""
 
@@ -112,7 +97,7 @@ def main():
     installed_filter = re.compile(
         '|'.join('(?:' + p + ')' for p in installed_filter))
 
-    manifest = _load_manifest_from_args(args)
+    manifest = ninja.load_manifest_from_args(args)
 
     # Build lookup map
     graph = {}

vndk/tools/sourcedr/sourcedr/ninja/ninja.py

@@ -1018,6 +1018,15 @@ def _parse_args():
 
 def load_manifest_from_args(args):
     """Load the input manifest specified by command line options."""
+
+    input_file = args.input_file
+
+    # If the input file name ends with `.pickle`, load it with pickle.load().
+    if input_file.endswith('.pickle'):
+        with open(input_file, 'rb') as pickle_file:
+            return pickle.load(pickle_file)
+
+    # Parse the ninja file
     return Parser(args.cwd).parse(args.input_file, args.encoding,
                                   args.ninja_deps)
 

vndk/tools/sourcedr/sourcedr/ninja/tests/run_tests.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -ex
+
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cd "$(dirname "$0")/.."
+
+python3 -m unittest discover "$@"
+python -m unittest discover "$@"

vndk/tools/sourcedr/sourcedr/ninja/tests/test_ninja.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
-from sourcedr import ninja
+import ninja
 
 import os
 import unittest
 
 
 TEST_DIR = os.path.abspath(os.path.dirname(__file__))
-TEST_DATA_DIR = os.path.join(TEST_DIR, 'testdata', 'ninja')
+TEST_DATA_DIR = os.path.join(TEST_DIR, 'testdata')
 
 ENCODING = 'utf-8'