Merge "Prevent manifest file from being added to prebuilts" into pi-dev

vndk/snapshot/check_gpl_license.py

@@ -18,7 +18,9 @@
 import argparse
 import glob
 import os
+import shutil
 import subprocess
+import tempfile
 import xml.etree.ElementTree as xml_tree
 
 import utils
@@ -33,7 +35,7 @@ class GPLChecker(object):
     MANIFEST_XML = utils.MANIFEST_FILE_NAME
     MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
 
-    def __init__(self, install_dir, android_build_top):
+    def __init__(self, install_dir, android_build_top, temp_artifact_dir):
         """GPLChecker constructor.
 
         Args:
@@ -43,14 +45,16 @@ class GPLChecker(object):
         """
         self._android_build_top = android_build_top
         self._install_dir = install_dir
-        self._manifest_file = os.path.join(install_dir,
+        self._manifest_file = os.path.join(temp_artifact_dir,
-                                           utils.MANIFEST_FILE_PATH)
+                                           self.MANIFEST_XML)
         self._notice_files_dir = os.path.join(install_dir,
                                               utils.NOTICE_FILES_DIR_PATH)
 
         if not os.path.isfile(self._manifest_file):
-            raise RuntimeError('{manifest} not found at {manifest_file}'.format(
+            raise RuntimeError(
-                manifest=self.MANIFEST_XML, manifest_file=self._manifest_file))
+                '{manifest} not found at {manifest_file}'.format(
+                    manifest=self.MANIFEST_XML,
+                    manifest_file=self._manifest_file))
 
     def _parse_module_paths(self):
         """Parses the module_path.txt files into a dictionary,
@@ -115,8 +119,8 @@ class GPLChecker(object):
 
         notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
         if len(notice_files) == 0:
-            raise RuntimeError(
+            raise RuntimeError('No license files found in {}'.format(
-                'No license files found in {}'.format(self._notice_files_dir))
+                self._notice_files_dir))
 
         gpl_projects = []
         pattern = 'GENERAL PUBLIC LICENSE'
@@ -173,6 +177,8 @@ def get_args():
     parser.add_argument(
         'vndk_version', type=int,
         help='VNDK snapshot version to check, e.g. "27".')
+    parser.add_argument('-b', '--branch', help='Branch to pull manifest from.')
+    parser.add_argument('--build', help='Build number to pull manifest from.')
     return parser.parse_args()
 
 
@@ -194,12 +200,25 @@ def main():
             'Please provide valid VNDK version. {} does not exist.'
             .format(install_dir))
 
-    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
+    temp_artifact_dir = tempfile.mkdtemp()
+    os.chdir(temp_artifact_dir)
+    manifest_pattern = 'manifest_{}.xml'.format(args.build)
+    print 'Fetching {file} from {branch} (bid: {build})'.format(
+        file=manifest_pattern, branch=args.branch, build=args.build)
+    manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME)
+    utils.fetch_artifact(args.branch, args.build, manifest_pattern,
+                         manifest_dest)
+
+    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
+                                 temp_artifact_dir)
     try:
         license_checker.check_gpl_projects()
     except ValueError as error:
         print error
         raise
+    finally:
+        print 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)
+        shutil.rmtree(temp_artifact_dir)
 
 
 if __name__ == '__main__':

vndk/snapshot/gen_buildfiles.py

@@ -45,7 +45,6 @@ class GenBuildFile(object):
         ... (other {SNAPSHOT_VARIANT}/ directories)
         common/
             Android.mk
-            manifest.xml
             NOTICE_FILES/
                 (license files, e.g. libfoo.so.txt)
     """
@@ -297,13 +296,13 @@ class GenBuildFile(object):
             variant_path = os.path.join(self._install_dir, variant)
             src_paths = utils.find(variant_path, [prebuilt])
             for src in sorted(src_paths):
-                arch_srcs += (
+                arch_srcs += ('{ind}{ind}{arch}: {{\n'
-                    '{ind}{ind}{arch}: {{\n'
+                              '{ind}{ind}{ind}srcs: ["{src}"],\n'
-                    '{ind}{ind}{ind}srcs: ["{src}"],\n'
+                              '{ind}{ind}}},\n'.format(
-                    '{ind}{ind}}},\n'.format(
+                                  ind=self.INDENT,
-                        ind=self.INDENT,
+                                  arch=utils.arch_from_path(
-                        arch=utils.arch_from_path(os.path.join(variant, src)),
+                                      os.path.join(variant, src)),
-                        src=src))
+                                  src=src))
             arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
             return arch_srcs
 

vndk/snapshot/update.py

@@ -46,15 +46,6 @@ def check_call(cmd):
     subprocess.check_call(cmd)
 
 
-def fetch_artifact(branch, build, pattern, destination='.'):
-    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
-    cmd = [
-        fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid',
-        build, pattern, destination
-    ]
-    check_call(cmd)
-
-
 def start_branch(build):
     branch_name = 'update-' + (build or 'local')
     logger().info('Creating branch {branch} in {dir}'.format(
@@ -75,7 +66,7 @@ def remove_old_snapshot(install_dir):
             sys.exit(1)
 
 
-def install_snapshot(branch, build, install_dir):
+def install_snapshot(branch, build, install_dir, temp_artifact_dir):
     """Installs VNDK snapshot build artifacts to prebuilts/vndk/v{version}.
 
     1) Fetch build artifacts from Android Build server or from local DIST_DIR
@@ -85,48 +76,38 @@ def install_snapshot(branch, build, install_dir):
       branch: string or None, branch name of build artifacts
       build: string or None, build number of build artifacts
       install_dir: string, directory to install VNDK snapshot
+      temp_artifact_dir: string, temp directory to hold build artifacts fetched
+        from Android Build server. For 'local' option, is set to None.
     """
     artifact_pattern = 'android-vndk-*.zip'
 
-    try:
+    if branch and build:
-        if branch and build:
+        artifact_dir = temp_artifact_dir
-            tempdir = tempfile.mkdtemp()
+        os.chdir(temp_artifact_dir)
-            artifact_dir = tempdir
+        logger().info('Fetching {pattern} from {branch} (bid: {build})'.format(
+            pattern=artifact_pattern, branch=branch, build=build))
+        utils.fetch_artifact(branch, build, artifact_pattern)
 
-            os.chdir(tempdir)
+        manifest_pattern = 'manifest_{}.xml'.format(build)
-            logger().info(
+        logger().info('Fetching {file} from {branch} (bid: {build})'.format(
-                'Fetching {pattern} from {branch} (bid: {build})'
+            file=manifest_pattern, branch=branch, build=build))
-                .format(pattern=artifact_pattern, branch=branch, build=build))
+        utils.fetch_artifact(branch, build, manifest_pattern,
-            fetch_artifact(branch, build, artifact_pattern)
+                             utils.MANIFEST_FILE_NAME)
 
-            manifest_pattern = 'manifest_{}.xml'.format(build)
+        os.chdir(install_dir)
-            manifest_name = utils.MANIFEST_FILE_NAME
+    else:
-            logger().info(
+        logger().info('Fetching local VNDK snapshot from {}'.format(DIST_DIR))
-                'Fetching {file} from {branch} (bid: {build})'.format(
+        artifact_dir = DIST_DIR
-                    file=manifest_pattern, branch=branch, build=build))
-            fetch_artifact(branch, build, manifest_pattern, manifest_name)
-            shutil.move(manifest_name,
-                        os.path.join(install_dir, utils.COMMON_DIR_PATH))
 
-            os.chdir(install_dir)
+    artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
-        else:
+    artifact_cnt = len(artifacts)
-            logger().info(
+    if artifact_cnt < 4:
-                'Fetching local VNDK snapshot from {}'.format(DIST_DIR))
+        raise RuntimeError(
-            artifact_dir = DIST_DIR
+            'Expected four android-vndk-*.zip files in {path}. Instead '
-
+            'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt))
-        artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
+    for artifact in artifacts:
-        artifact_cnt = len(artifacts)
+        logger().info('Unzipping VNDK snapshot: {}'.format(artifact))
-        if artifact_cnt < 4:
+        check_call(['unzip', '-q', artifact, '-d', install_dir])
-            raise RuntimeError(
-                'Expected four android-vndk-*.zip files in {path}. Instead '
-                'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt))
-        for artifact in artifacts:
-            logger().info('Unzipping VNDK snapshot: {}'.format(artifact))
-            check_call(['unzip', '-q', artifact, '-d', install_dir])
-    finally:
-        if branch and build:
-            logger().info('Deleting tempdir: {}'.format(tempdir))
-            shutil.rmtree(tempdir)
 
 
 def gather_notice_files(install_dir):
@@ -255,16 +236,35 @@ def main():
 
     remove_old_snapshot(install_dir)
     os.makedirs(utils.COMMON_DIR_PATH)
-    install_snapshot(args.branch, args.build, install_dir)
-    gather_notice_files(install_dir)
-    revise_ld_config_txt_if_needed(vndk_version)
-
-    buildfile_generator = GenBuildFile(install_dir, vndk_version)
-    update_buildfiles(buildfile_generator)
 
+    temp_artifact_dir = None
     if not args.local:
-        license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
+        temp_artifact_dir = tempfile.mkdtemp()
-        check_gpl_license(license_checker)
+
+    install_status = True
+    try:
+        install_snapshot(args.branch, args.build, install_dir,
+                         temp_artifact_dir)
+        gather_notice_files(install_dir)
+        revise_ld_config_txt_if_needed(vndk_version)
+
+        buildfile_generator = GenBuildFile(install_dir, vndk_version)
+        update_buildfiles(buildfile_generator)
+
+        if not args.local:
+            license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
+                                         temp_artifact_dir)
+            check_gpl_license(license_checker)
+    except:
+        logger().info('FAILED TO INSTALL SNAPSHOT')
+        install_status = False
+    finally:
+        if temp_artifact_dir:
+            logger().info(
+                'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
+            shutil.rmtree(temp_artifact_dir)
+
+    if not args.local and install_status:
         commit(args.branch, args.build, vndk_version)
 
 

vndk/snapshot/utils.py

@@ -19,6 +19,7 @@
 import glob
 import os
 import re
+import subprocess
 import sys
 
 # Global Keys
@@ -28,7 +29,6 @@ COMMON_DIR_PATH = COMMON_DIR_NAME
 ANDROID_MK_PATH = os.path.join(COMMON_DIR_PATH, 'Android.mk')
 CONFIG_DIR_PATH_PATTERN = '*/configs'
 MANIFEST_FILE_NAME = 'manifest.xml'
-MANIFEST_FILE_PATH = os.path.join(COMMON_DIR_PATH, MANIFEST_FILE_NAME)
 MODULE_PATHS_FILE_NAME = 'module_paths.txt'
 NOTICE_FILES_DIR_NAME = 'NOTICE_FILES'
 NOTICE_FILES_DIR_PATH = os.path.join(COMMON_DIR_PATH, NOTICE_FILES_DIR_NAME)
@@ -113,3 +113,21 @@ def find(path, names):
                 rel_to_root = abspath.replace(os.path.abspath(path), '')
                 found.append(rel_to_root[1:])  # strip leading /
     return found
+
+
+def fetch_artifact(branch, build, pattern, destination='.'):
+    """Fetches build artifacts from Android Build server.
+
+    Args:
+      branch: string, branch to pull build artifacts from
+      build: string, build number to pull build artifacts from
+      pattern: string, pattern of build artifact file name
+      destination: string, destination to pull build artifact to
+    """
+    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
+    cmd = [
+        fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid',
+        build, pattern, destination
+    ]
+    print 'Running `{}`'.format(' '.join(cmd))
+    subprocess.check_call(cmd)