Merge "Add script to check GPL projects have released SHA"

vndk/snapshot/check_gpl_license.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import glob
+import os
+import subprocess
+import xml.etree.ElementTree as xml_tree
+
+import utils
+
+
+class GPLChecker(object):
+    """Checks that all GPL projects in a VNDK snapshot have released sources.
+
+    Makes sure that the current source tree have the sources for all GPL
+    prebuilt libraries in a specified VNDK snapshot version.
+    """
+    MANIFEST_XML = 'manifest.xml'
+    MODULE_PATHS_TXT = 'module_paths.txt'
+
+    def __init__(self, install_dir, android_build_top):
+        """GPLChecker constructor.
+
+        Args:
+          install_dir: string, absolute path to the prebuilts/vndk/v{version}
+            directory where the build files will be generated.
+          android_build_top: string, absolute path to ANDROID_BUILD_TOP
+        """
+        self._android_build_top = android_build_top
+        self._install_dir = install_dir
+        self._manifest_file = os.path.join(install_dir, self.MANIFEST_XML)
+        self._notice_files_dir = os.path.join(install_dir, 'NOTICE_FILES')
+
+        if not os.path.isfile(self._manifest_file):
+            raise RuntimeError('{manifest} not found in {install_dir}'.format(
+                manifest=self.MANIFEST_XML, install_dir=install_dir))
+
+    def _parse_module_paths(self):
+        """Parses the module_path.txt files into a dictionary,
+
+        Returns:
+          module_paths: dict, e.g. {libfoo.so: some/path/here}
+        """
+        module_paths = dict()
+        for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
+            file_path = os.path.join(self._install_dir, file)
+            with open(file_path, 'r') as f:
+                for line in f.read().strip().split('\n'):
+                    paths = line.split(' ')
+                    if len(paths) > 1:
+                        if paths[0] not in module_paths:
+                            module_paths[paths[0]] = paths[1]
+        return module_paths
+
+    def _parse_manifest(self):
+        """Parses manifest.xml file and returns list of 'project' tags."""
+
+        root = xml_tree.parse(self._manifest_file).getroot()
+        return root.findall('project')
+
+    def _get_revision(self, module_path, manifest_projects):
+        """Returns revision value recorded in manifest.xml for given project.
+
+        Args:
+          module_path: string, project path relative to ANDROID_BUILD_TOP
+          manifest_projects: list of xml_tree.Element, list of 'project' tags
+        """
+        revision = None
+        for project in manifest_projects:
+            path = project.get('path')
+            if module_path.startswith(path):
+                revision = project.get('revision')
+                break
+        return revision
+
+    def _check_revision_exists(self, revision, git_project_path):
+        """Checks whether a revision is found in a git project of current tree.
+
+        Args:
+          revision: string, revision value recorded in manifest.xml
+          git_project_path: string, path relative to ANDROID_BUILD_TOP
+        """
+        path = utils.join_realpath(self._android_build_top, git_project_path)
+        try:
+            subprocess.check_call(
+                ['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)])
+            return True
+        except subprocess.CalledProcessError:
+            return False
+
+    def check_gpl_projects(self):
+        """Checks that all GPL projects have released sources.
+
+        Raises:
+          ValueError: There are GPL projects with unreleased sources.
+        """
+        print 'Starting license check for GPL projects...'
+
+        notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
+        if len(notice_files) == 0:
+            raise RuntimeError(
+                'No license files found in {}'.format(self._notice_files_dir))
+
+        gpl_projects = []
+        pattern = 'GENERAL PUBLIC LICENSE'
+        for notice_file_path in notice_files:
+            with open(notice_file_path, 'r') as notice_file:
+                if pattern in notice_file.read():
+                    lib_name = os.path.splitext(
+                        os.path.basename(notice_file_path))[0]
+                    gpl_projects.append(lib_name)
+
+        if not gpl_projects:
+            print 'No GPL projects found.'
+            return
+
+        print 'GPL projects found:', ', '.join(gpl_projects)
+
+        module_paths = self._parse_module_paths()
+        manifest_projects = self._parse_manifest()
+        released_projects = []
+        unreleased_projects = []
+
+        for lib in gpl_projects:
+            if lib in module_paths:
+                module_path = module_paths[lib]
+                revision = self._get_revision(module_path, manifest_projects)
+                if not revision:
+                    raise RuntimeError(
+                        'No project found for {path} in {manifest}'.format(
+                            path=module_path, manifest=self.MANIFEST_XML))
+                revision_exists = self._check_revision_exists(
+                    revision, module_path)
+                if not revision_exists:
+                    unreleased_projects.append((lib, module_path))
+                else:
+                    released_projects.append((lib, module_path))
+            else:
+                raise RuntimeError(
+                    'No module path was found for {lib} in {module_paths}'.
+                    format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
+
+        if released_projects:
+            print 'Released GPL projects:', released_projects
+
+        if unreleased_projects:
+            raise ValueError(
+                ('FAIL: The following GPL projects have NOT been released in '
+                 'current tree: {}'.format(unreleased_projects)))
+
+        print 'PASS: All GPL projects have source in current tree.'
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'vndk_version', type=int,
+        help='VNDK snapshot version to check, e.g. "27".')
+    return parser.parse_args()
+
+
+def main():
+    """For local testing purposes.
+
+    Note: VNDK snapshot must be already installed under
+      prebuilts/vndk/v{version}.
+    """
+    ANDROID_BUILD_TOP = utils.get_android_build_top()
+    PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
+                                             'prebuilts/vndk')
+
+    args = get_args()
+    vndk_version = args.vndk_version
+    install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
+    if not os.path.isdir(install_dir):
+        raise ValueError(
+            'Please provide valid VNDK version. {} does not exist.'
+            .format(install_dir))
+
+    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
+    try:
+        license_checker.check_gpl_projects()
+    except ValueError as error:
+        print error
+        raise
+
+
+if __name__ == '__main__':
+    main()

vndk/snapshot/gen_buildfiles.py

@@ -18,6 +18,8 @@
 import os
 import sys
 
+import utils
+
 
 class GenBuildFile(object):
     """Generates Android.mk and Android.bp for prebuilts/vndk/v{version}."""
@@ -53,7 +55,7 @@ class GenBuildFile(object):
           txt_filename: string, file name in VNDK snapshot
         """
         prebuilt_list = []
-        txts = find(self._install_dir, [txt_filename])
+        txts = utils.find(self._install_dir, [txt_filename])
         for txt in txts:
             path_to_txt = os.path.join(self._install_dir, txt)
             with open(path_to_txt, 'r') as f:
@@ -126,7 +128,7 @@ class GenBuildFile(object):
         Args:
           prebuilt: string, name of ETC prebuilt object
         """
-        etc_path = find(self._install_dir, prebuilt)[0]
+        etc_path = utils.find(self._install_dir, prebuilt)[0]
         etc_sub_path = etc_path[etc_path.index('/') + 1:]
 
         return (
@@ -202,7 +204,7 @@ class GenBuildFile(object):
             notice = ''
             notice_file_name = '{}.txt'.format(prebuilt)
             notices_dir = os.path.join(self._install_dir, 'NOTICE_FILES')
-            notice_files = find(notices_dir, [notice_file_name])
+            notice_files = utils.find(notices_dir, [notice_file_name])
             if len(notice_files) > 0:
                 notice = '{ind}notice: "{notice_file_path}",\n'.format(
                     ind=self.INDENT,
@@ -241,15 +243,13 @@ class GenBuildFile(object):
               prebuilt: string, name of prebuilt object
             """
             arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT)
-            src_paths = find(self._install_dir, [prebuilt])
+            src_paths = utils.find(self._install_dir, [prebuilt])
-            # if len(src_paths) < 4:
-            #     print prebuilt, src_paths
             for src in sorted(src_paths):
                 arch_srcs += ('{ind}{ind}{arch}: {{\n'
                               '{ind}{ind}{ind}srcs: ["{src}"],\n'
                               '{ind}{ind}}},\n'.format(
                                   ind=self.INDENT,
-                                  arch=arch_from_path(src),
+                                  arch=utils.arch_from_path(src),
                                   src=src))
             arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
             return arch_srcs
@@ -287,48 +287,15 @@ class GenBuildFile(object):
                     arch_srcs=arch_srcs))
 
 
-def find(path, names):
-    """Finds a list of files in a directory that match the given names.
-
-    Args:
-      path: string, absolute path of directory from which to find files
-      names: list of strings, names of the files to find
-    """
-    found = []
-    for root, _, files in os.walk(path):
-        for file_name in sorted(files):
-            if file_name in names:
-                abspath = os.path.abspath(os.path.join(root, file_name))
-                rel_to_root = abspath.replace(os.path.abspath(path), '')
-                found.append(rel_to_root[1:])  # strip leading /
-    return found
-
-
-def arch_from_path(path):
-    """Extracts archfrom given VNDK snapshot path.
-
-    Args:
-      path: string, path relative to prebuilts/vndk/v{version}
-
-    Returns:
-      arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64")
-    """
-    return path.split('/')[0].split('-')[1]
-
-
 def main():
     """For local testing purposes.
 
     Note: VNDK snapshot must be already installed under
       prebuilts/vndk/v{version}.
     """
-    ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
+    ANDROID_BUILD_TOP = utils.get_android_build_top()
-    if not ANDROID_BUILD_TOP:
+    PREBUILTS_VNDK_DIR = utils.join_realpath(
-        print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
+        ANDROID_BUILD_TOP, 'prebuilts/vndk')
-              '\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
-        sys.exit(1)
-    PREBUILTS_VNDK_DIR = os.path.realpath(
-        os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk'))
 
     vndk_version = 27  # set appropriately
     install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))

vndk/snapshot/update.py

@@ -26,24 +26,14 @@ import sys
 import tempfile
 import textwrap
 
+import utils
+
+from check_gpl_license import GPLChecker
 from gen_buildfiles import GenBuildFile
 
-ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
+ANDROID_BUILD_TOP = utils.get_android_build_top()
-if not ANDROID_BUILD_TOP:
+DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP))
-    print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
+PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk')
-          '\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
-    sys.exit(1)
-
-DIST_DIR = os.getenv('DIST_DIR')
-if not DIST_DIR:
-    OUT_DIR = os.getenv('OUT_DIR')
-    if OUT_DIR:
-        DIST_DIR = os.path.realpath(os.path.join(OUT_DIR, 'dist'))
-    else:
-        DIST_DIR = os.path.realpath(os.path.join(ANDROID_BUILD_TOP, 'out/dist'))
-
-PREBUILTS_VNDK_DIR = os.path.realpath(
-    os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk'))
 
 
 def logger():
@@ -103,21 +93,23 @@ def install_snapshot(branch, build, install_dir):
             artifact_dir = tempdir
 
             os.chdir(tempdir)
-            logger().info('Fetching {pattern} from {branch} (bid: {build})'
+            logger().info(
+                'Fetching {pattern} from {branch} (bid: {build})'
                 .format(pattern=artifact_pattern, branch=branch, build=build))
             fetch_artifact(branch, build, artifact_pattern)
 
             manifest_pattern = 'manifest_{}.xml'.format(build)
             manifest_name = 'manifest.xml'
-            logger().info('Fetching {file} from {branch} (bid: {build})'.format(
+            logger().info(
-                file=manifest_pattern, branch=branch, build=build))
+                'Fetching {file} from {branch} (bid: {build})'.format(
+                    file=manifest_pattern, branch=branch, build=build))
             fetch_artifact(branch, build, manifest_pattern, manifest_name)
             shutil.move(manifest_name, install_dir)
 
             os.chdir(install_dir)
         else:
-            logger().info('Fetching local VNDK snapshot from {}'.format(
+            logger().info(
-                DIST_DIR))
+                'Fetching local VNDK snapshot from {}'.format(DIST_DIR))
             artifact_dir = DIST_DIR
 
         artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
@@ -148,9 +140,10 @@ def gather_notice_files():
         notices_dir_per_arch = os.path.join(arch_dir, notices_dir_name)
         if os.path.isdir(notices_dir_per_arch):
             for notice_file in glob.glob(
-                '{}/*.txt'.format(notices_dir_per_arch)):
+                    '{}/*.txt'.format(notices_dir_per_arch)):
-                if not os.path.isfile(os.path.join(notices_dir_name,
+                if not os.path.isfile(
-                    os.path.basename(notice_file))):
+                        os.path.join(notices_dir_name,
+                                     os.path.basename(notice_file))):
                     shutil.copy(notice_file, notices_dir_name)
             shutil.rmtree(notices_dir_per_arch)
 
@@ -163,6 +156,14 @@ def update_buildfiles(buildfile_generator):
     buildfile_generator.generate_android_bp()
 
 
+def check_gpl_license(license_checker):
+    try:
+        license_checker.check_gpl_projects()
+    except ValueError as error:
+        print '***CANNOT INSTALL VNDK SNAPSHOT***', error
+        raise
+
+
 def commit(branch, build, version):
     logger().info('Making commit...')
     check_call(['git', 'add', '.'])
@@ -186,7 +187,7 @@ def get_args():
         help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of '
               'Android Build server.'))
     parser.add_argument(
-        '--use-current-branch',action='store_true',
+        '--use-current-branch', action='store_true',
         help='Perform the update in the current branch. Do not repo start.')
     parser.add_argument(
         '-v', '--verbose', action='count', default=0,
@@ -239,6 +240,8 @@ def main():
     update_buildfiles(buildfile_generator)
 
     if not args.local:
+        license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
+        check_gpl_license(license_checker)
         commit(args.branch, args.build, vndk_version)
 
 

vndk/snapshot/utils.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utility functions for VNDK snapshot."""
+
+import os
+import sys
+
+
+def get_android_build_top():
+    ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
+    if not ANDROID_BUILD_TOP:
+        print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
+              '\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
+        sys.exit(1)
+    return ANDROID_BUILD_TOP
+
+
+def join_realpath(root, *args):
+    return os.path.realpath(os.path.join(root, *args))
+
+
+def _get_dir_from_env(env_var, default):
+    return os.path.realpath(os.getenv(env_var, default))
+
+
+def get_out_dir(android_build_top):
+    return _get_dir_from_env('OUT_DIR', join_realpath(android_build_top,
+                                                      'out'))
+
+
+def get_dist_dir(out_dir):
+    return _get_dir_from_env('DIST_DIR', join_realpath(out_dir, 'dist'))
+
+
+def arch_from_path(path):
+    """Extracts arch from given VNDK snapshot path.
+
+    Args:
+      path: string, path relative to prebuilts/vndk/v{version}
+
+    Returns:
+      arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64")
+    """
+    return path.split('/')[0].split('-')[1]
+
+
+def find(path, names):
+    """Returns a list of files in a directory that match the given names.
+
+    Args:
+      path: string, absolute path of directory from which to find files
+      names: list of strings, names of the files to find
+    """
+    found = []
+    for root, _, files in os.walk(path):
+        for file_name in sorted(files):
+            if file_name in names:
+                abspath = os.path.abspath(os.path.join(root, file_name))
+                rel_to_root = abspath.replace(os.path.abspath(path), '')
+                found.append(rel_to_root[1:])  # strip leading /
+    return found