diff --git a/vndk/snapshot/check_gpl_license.py b/vndk/snapshot/check_gpl_license.py new file mode 100644 index 000000000..228888e52 --- /dev/null +++ b/vndk/snapshot/check_gpl_license.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import glob +import os +import subprocess +import xml.etree.ElementTree as xml_tree + +import utils + + +class GPLChecker(object): + """Checks that all GPL projects in a VNDK snapshot have released sources. + + Makes sure that the current source tree have the sources for all GPL + prebuilt libraries in a specified VNDK snapshot version. + """ + MANIFEST_XML = 'manifest.xml' + MODULE_PATHS_TXT = 'module_paths.txt' + + def __init__(self, install_dir, android_build_top): + """GPLChecker constructor. + + Args: + install_dir: string, absolute path to the prebuilts/vndk/v{version} + directory where the build files will be generated. + android_build_top: string, absolute path to ANDROID_BUILD_TOP + """ + self._android_build_top = android_build_top + self._install_dir = install_dir + self._manifest_file = os.path.join(install_dir, self.MANIFEST_XML) + self._notice_files_dir = os.path.join(install_dir, 'NOTICE_FILES') + + if not os.path.isfile(self._manifest_file): + raise RuntimeError('{manifest} not found in {install_dir}'.format( + manifest=self.MANIFEST_XML, install_dir=install_dir)) + + def _parse_module_paths(self): + """Parses the module_path.txt files into a dictionary, + + Returns: + module_paths: dict, e.g. {libfoo.so: some/path/here} + """ + module_paths = dict() + for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]): + file_path = os.path.join(self._install_dir, file) + with open(file_path, 'r') as f: + for line in f.read().strip().split('\n'): + paths = line.split(' ') + if len(paths) > 1: + if paths[0] not in module_paths: + module_paths[paths[0]] = paths[1] + return module_paths + + def _parse_manifest(self): + """Parses manifest.xml file and returns list of 'project' tags.""" + + root = xml_tree.parse(self._manifest_file).getroot() + return root.findall('project') + + def _get_revision(self, module_path, manifest_projects): + """Returns revision value recorded in manifest.xml for given project. + + Args: + module_path: string, project path relative to ANDROID_BUILD_TOP + manifest_projects: list of xml_tree.Element, list of 'project' tags + """ + revision = None + for project in manifest_projects: + path = project.get('path') + if module_path.startswith(path): + revision = project.get('revision') + break + return revision + + def _check_revision_exists(self, revision, git_project_path): + """Checks whether a revision is found in a git project of current tree. + + Args: + revision: string, revision value recorded in manifest.xml + git_project_path: string, path relative to ANDROID_BUILD_TOP + """ + path = utils.join_realpath(self._android_build_top, git_project_path) + try: + subprocess.check_call( + ['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)]) + return True + except subprocess.CalledProcessError: + return False + + def check_gpl_projects(self): + """Checks that all GPL projects have released sources. + + Raises: + ValueError: There are GPL projects with unreleased sources. + """ + print 'Starting license check for GPL projects...' + + notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) + if len(notice_files) == 0: + raise RuntimeError( + 'No license files found in {}'.format(self._notice_files_dir)) + + gpl_projects = [] + pattern = 'GENERAL PUBLIC LICENSE' + for notice_file_path in notice_files: + with open(notice_file_path, 'r') as notice_file: + if pattern in notice_file.read(): + lib_name = os.path.splitext( + os.path.basename(notice_file_path))[0] + gpl_projects.append(lib_name) + + if not gpl_projects: + print 'No GPL projects found.' + return + + print 'GPL projects found:', ', '.join(gpl_projects) + + module_paths = self._parse_module_paths() + manifest_projects = self._parse_manifest() + released_projects = [] + unreleased_projects = [] + + for lib in gpl_projects: + if lib in module_paths: + module_path = module_paths[lib] + revision = self._get_revision(module_path, manifest_projects) + if not revision: + raise RuntimeError( + 'No project found for {path} in {manifest}'.format( + path=module_path, manifest=self.MANIFEST_XML)) + revision_exists = self._check_revision_exists( + revision, module_path) + if not revision_exists: + unreleased_projects.append((lib, module_path)) + else: + released_projects.append((lib, module_path)) + else: + raise RuntimeError( + 'No module path was found for {lib} in {module_paths}'. + format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) + + if released_projects: + print 'Released GPL projects:', released_projects + + if unreleased_projects: + raise ValueError( + ('FAIL: The following GPL projects have NOT been released in ' + 'current tree: {}'.format(unreleased_projects))) + + print 'PASS: All GPL projects have source in current tree.' + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'vndk_version', type=int, + help='VNDK snapshot version to check, e.g. "27".') + return parser.parse_args() + + +def main(): + """For local testing purposes. + + Note: VNDK snapshot must be already installed under + prebuilts/vndk/v{version}. + """ + ANDROID_BUILD_TOP = utils.get_android_build_top() + PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, + 'prebuilts/vndk') + + args = get_args() + vndk_version = args.vndk_version + install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) + if not os.path.isdir(install_dir): + raise ValueError( + 'Please provide valid VNDK version. {} does not exist.' + .format(install_dir)) + + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) + try: + license_checker.check_gpl_projects() + except ValueError as error: + print error + raise + + +if __name__ == '__main__': + main() diff --git a/vndk/snapshot/gen_buildfiles.py b/vndk/snapshot/gen_buildfiles.py index 23bd97682..ba238ba8b 100644 --- a/vndk/snapshot/gen_buildfiles.py +++ b/vndk/snapshot/gen_buildfiles.py @@ -18,6 +18,8 @@ import os import sys +import utils + class GenBuildFile(object): """Generates Android.mk and Android.bp for prebuilts/vndk/v{version}.""" @@ -53,7 +55,7 @@ class GenBuildFile(object): txt_filename: string, file name in VNDK snapshot """ prebuilt_list = [] - txts = find(self._install_dir, [txt_filename]) + txts = utils.find(self._install_dir, [txt_filename]) for txt in txts: path_to_txt = os.path.join(self._install_dir, txt) with open(path_to_txt, 'r') as f: @@ -126,7 +128,7 @@ class GenBuildFile(object): Args: prebuilt: string, name of ETC prebuilt object """ - etc_path = find(self._install_dir, prebuilt)[0] + etc_path = utils.find(self._install_dir, prebuilt)[0] etc_sub_path = etc_path[etc_path.index('/') + 1:] return ( @@ -202,7 +204,7 @@ class GenBuildFile(object): notice = '' notice_file_name = '{}.txt'.format(prebuilt) notices_dir = os.path.join(self._install_dir, 'NOTICE_FILES') - notice_files = find(notices_dir, [notice_file_name]) + notice_files = utils.find(notices_dir, [notice_file_name]) if len(notice_files) > 0: notice = '{ind}notice: "{notice_file_path}",\n'.format( ind=self.INDENT, @@ -241,15 +243,13 @@ class GenBuildFile(object): prebuilt: string, name of prebuilt object """ arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT) - src_paths = find(self._install_dir, [prebuilt]) - # if len(src_paths) < 4: - # print prebuilt, src_paths + src_paths = utils.find(self._install_dir, [prebuilt]) for src in sorted(src_paths): arch_srcs += ('{ind}{ind}{arch}: {{\n' '{ind}{ind}{ind}srcs: ["{src}"],\n' '{ind}{ind}}},\n'.format( ind=self.INDENT, - arch=arch_from_path(src), + arch=utils.arch_from_path(src), src=src)) arch_srcs += '{ind}}},\n'.format(ind=self.INDENT) return arch_srcs @@ -287,48 +287,15 @@ class GenBuildFile(object): arch_srcs=arch_srcs)) -def find(path, names): - """Finds a list of files in a directory that match the given names. - - Args: - path: string, absolute path of directory from which to find files - names: list of strings, names of the files to find - """ - found = [] - for root, _, files in os.walk(path): - for file_name in sorted(files): - if file_name in names: - abspath = os.path.abspath(os.path.join(root, file_name)) - rel_to_root = abspath.replace(os.path.abspath(path), '') - found.append(rel_to_root[1:]) # strip leading / - return found - - -def arch_from_path(path): - """Extracts archfrom given VNDK snapshot path. - - Args: - path: string, path relative to prebuilts/vndk/v{version} - - Returns: - arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64") - """ - return path.split('/')[0].split('-')[1] - - def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ - ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') - if not ANDROID_BUILD_TOP: - print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' - '\'. build/envsetup.sh; lunch \'. Exiting script.') - sys.exit(1) - PREBUILTS_VNDK_DIR = os.path.realpath( - os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk')) + ANDROID_BUILD_TOP = utils.get_android_build_top() + PREBUILTS_VNDK_DIR = utils.join_realpath( + ANDROID_BUILD_TOP, 'prebuilts/vndk') vndk_version = 27 # set appropriately install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) diff --git a/vndk/snapshot/update.py b/vndk/snapshot/update.py index e10a5760c..1038a9985 100644 --- a/vndk/snapshot/update.py +++ b/vndk/snapshot/update.py @@ -26,24 +26,14 @@ import sys import tempfile import textwrap +import utils + +from check_gpl_license import GPLChecker from gen_buildfiles import GenBuildFile -ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') -if not ANDROID_BUILD_TOP: - print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' - '\'. build/envsetup.sh; lunch \'. Exiting script.') - sys.exit(1) - -DIST_DIR = os.getenv('DIST_DIR') -if not DIST_DIR: - OUT_DIR = os.getenv('OUT_DIR') - if OUT_DIR: - DIST_DIR = os.path.realpath(os.path.join(OUT_DIR, 'dist')) - else: - DIST_DIR = os.path.realpath(os.path.join(ANDROID_BUILD_TOP, 'out/dist')) - -PREBUILTS_VNDK_DIR = os.path.realpath( - os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk')) +ANDROID_BUILD_TOP = utils.get_android_build_top() +DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP)) +PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') def logger(): @@ -103,21 +93,23 @@ def install_snapshot(branch, build, install_dir): artifact_dir = tempdir os.chdir(tempdir) - logger().info('Fetching {pattern} from {branch} (bid: {build})' + logger().info( + 'Fetching {pattern} from {branch} (bid: {build})' .format(pattern=artifact_pattern, branch=branch, build=build)) fetch_artifact(branch, build, artifact_pattern) manifest_pattern = 'manifest_{}.xml'.format(build) manifest_name = 'manifest.xml' - logger().info('Fetching {file} from {branch} (bid: {build})'.format( - file=manifest_pattern, branch=branch, build=build)) + logger().info( + 'Fetching {file} from {branch} (bid: {build})'.format( + file=manifest_pattern, branch=branch, build=build)) fetch_artifact(branch, build, manifest_pattern, manifest_name) shutil.move(manifest_name, install_dir) os.chdir(install_dir) else: - logger().info('Fetching local VNDK snapshot from {}'.format( - DIST_DIR)) + logger().info( + 'Fetching local VNDK snapshot from {}'.format(DIST_DIR)) artifact_dir = DIST_DIR artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) @@ -148,9 +140,10 @@ def gather_notice_files(): notices_dir_per_arch = os.path.join(arch_dir, notices_dir_name) if os.path.isdir(notices_dir_per_arch): for notice_file in glob.glob( - '{}/*.txt'.format(notices_dir_per_arch)): - if not os.path.isfile(os.path.join(notices_dir_name, - os.path.basename(notice_file))): + '{}/*.txt'.format(notices_dir_per_arch)): + if not os.path.isfile( + os.path.join(notices_dir_name, + os.path.basename(notice_file))): shutil.copy(notice_file, notices_dir_name) shutil.rmtree(notices_dir_per_arch) @@ -163,6 +156,14 @@ def update_buildfiles(buildfile_generator): buildfile_generator.generate_android_bp() +def check_gpl_license(license_checker): + try: + license_checker.check_gpl_projects() + except ValueError as error: + print '***CANNOT INSTALL VNDK SNAPSHOT***', error + raise + + def commit(branch, build, version): logger().info('Making commit...') check_call(['git', 'add', '.']) @@ -186,7 +187,7 @@ def get_args(): help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of ' 'Android Build server.')) parser.add_argument( - '--use-current-branch',action='store_true', + '--use-current-branch', action='store_true', help='Perform the update in the current branch. Do not repo start.') parser.add_argument( '-v', '--verbose', action='count', default=0, @@ -239,6 +240,8 @@ def main(): update_buildfiles(buildfile_generator) if not args.local: + license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP) + check_gpl_license(license_checker) commit(args.branch, args.build, vndk_version) diff --git a/vndk/snapshot/utils.py b/vndk/snapshot/utils.py new file mode 100644 index 000000000..b21225b26 --- /dev/null +++ b/vndk/snapshot/utils.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utility functions for VNDK snapshot.""" + +import os +import sys + + +def get_android_build_top(): + ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') + if not ANDROID_BUILD_TOP: + print('Error: Missing ANDROID_BUILD_TOP env variable. Please run ' + '\'. build/envsetup.sh; lunch \'. Exiting script.') + sys.exit(1) + return ANDROID_BUILD_TOP + + +def join_realpath(root, *args): + return os.path.realpath(os.path.join(root, *args)) + + +def _get_dir_from_env(env_var, default): + return os.path.realpath(os.getenv(env_var, default)) + + +def get_out_dir(android_build_top): + return _get_dir_from_env('OUT_DIR', join_realpath(android_build_top, + 'out')) + + +def get_dist_dir(out_dir): + return _get_dir_from_env('DIST_DIR', join_realpath(out_dir, 'dist')) + + +def arch_from_path(path): + """Extracts arch from given VNDK snapshot path. + + Args: + path: string, path relative to prebuilts/vndk/v{version} + + Returns: + arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64") + """ + return path.split('/')[0].split('-')[1] + + +def find(path, names): + """Returns a list of files in a directory that match the given names. + + Args: + path: string, absolute path of directory from which to find files + names: list of strings, names of the files to find + """ + found = [] + for root, _, files in os.walk(path): + for file_name in sorted(files): + if file_name in names: + abspath = os.path.abspath(os.path.join(root, file_name)) + rel_to_root = abspath.replace(os.path.abspath(path), '') + found.append(rel_to_root[1:]) # strip leading / + return found