LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Add script to check GPL projects have released SHA" am: 38fa0799f2 am: bae0c813a8

Browse Source 
am: 662a8a15db

Change-Id: I7cbb32fc9f87006cce496e25e3c31a632c4351f7
This commit is contained in:
Jae Shin
2017-12-21 20:24:19 +00:00
committed by android-build-merger
parent 5c3ec8d78b 662a8a15db
commit f7f7ea2530
4 changed files with 317 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
vndk/snapshot/check_gpl_license.py Normal file
View File

@@ -0,0 +1,204 @@
#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import glob
import os
import subprocess
import xml.etree.ElementTree as xml_tree
import utils
class GPLChecker(object):
"""Checks that all GPL projects in a VNDK snapshot have released sources.
Makes sure that the current source tree have the sources for all GPL
prebuilt libraries in a specified VNDK snapshot version.
"""
MANIFEST_XML = 'manifest.xml'
MODULE_PATHS_TXT = 'module_paths.txt'
def __init__(self, install_dir, android_build_top):
"""GPLChecker constructor.
Args:
install_dir: string, absolute path to the prebuilts/vndk/v{version}
directory where the build files will be generated.
android_build_top: string, absolute path to ANDROID_BUILD_TOP
"""
self._android_build_top = android_build_top
self._install_dir = install_dir
self._manifest_file = os.path.join(install_dir, self.MANIFEST_XML)
self._notice_files_dir = os.path.join(install_dir, 'NOTICE_FILES')
if not os.path.isfile(self._manifest_file):
raise RuntimeError('{manifest} not found in {install_dir}'.format(
manifest=self.MANIFEST_XML, install_dir=install_dir))
def _parse_module_paths(self):
"""Parses the module_path.txt files into a dictionary,
Returns:
module_paths: dict, e.g. {libfoo.so: some/path/here}
"""
module_paths = dict()
for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
file_path = os.path.join(self._install_dir, file)
with open(file_path, 'r') as f:
for line in f.read().strip().split('\n'):
paths = line.split(' ')
if len(paths) > 1:
if paths[0] not in module_paths:
module_paths[paths[0]] = paths[1]
return module_paths
def _parse_manifest(self):
"""Parses manifest.xml file and returns list of 'project' tags."""
root = xml_tree.parse(self._manifest_file).getroot()
return root.findall('project')
def _get_revision(self, module_path, manifest_projects):
"""Returns revision value recorded in manifest.xml for given project.
Args:
module_path: string, project path relative to ANDROID_BUILD_TOP
manifest_projects: list of xml_tree.Element, list of 'project' tags
"""
revision = None
for project in manifest_projects:
path = project.get('path')
if module_path.startswith(path):
revision = project.get('revision')
break
return revision
def _check_revision_exists(self, revision, git_project_path):
"""Checks whether a revision is found in a git project of current tree.
Args:
revision: string, revision value recorded in manifest.xml
git_project_path: string, path relative to ANDROID_BUILD_TOP
"""
path = utils.join_realpath(self._android_build_top, git_project_path)
try:
subprocess.check_call(
['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)])
return True
except subprocess.CalledProcessError:
return False
def check_gpl_projects(self):
"""Checks that all GPL projects have released sources.
Raises:
ValueError: There are GPL projects with unreleased sources.
"""
print 'Starting license check for GPL projects...'
notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
if len(notice_files) == 0:
raise RuntimeError(
'No license files found in {}'.format(self._notice_files_dir))
gpl_projects = []
pattern = 'GENERAL PUBLIC LICENSE'
for notice_file_path in notice_files:
with open(notice_file_path, 'r') as notice_file:
if pattern in notice_file.read():
lib_name = os.path.splitext(
os.path.basename(notice_file_path))[0]
gpl_projects.append(lib_name)
if not gpl_projects:
print 'No GPL projects found.'
return
print 'GPL projects found:', ', '.join(gpl_projects)
module_paths = self._parse_module_paths()
manifest_projects = self._parse_manifest()
released_projects = []
unreleased_projects = []
for lib in gpl_projects:
if lib in module_paths:
module_path = module_paths[lib]
revision = self._get_revision(module_path, manifest_projects)
if not revision:
raise RuntimeError(
'No project found for {path} in {manifest}'.format(
path=module_path, manifest=self.MANIFEST_XML))
revision_exists = self._check_revision_exists(
revision, module_path)
if not revision_exists:
unreleased_projects.append((lib, module_path))
else:
released_projects.append((lib, module_path))
else:
raise RuntimeError(
'No module path was found for {lib} in {module_paths}'.
format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
if released_projects:
print 'Released GPL projects:', released_projects
if unreleased_projects:
raise ValueError(
('FAIL: The following GPL projects have NOT been released in '
'current tree: {}'.format(unreleased_projects)))
print 'PASS: All GPL projects have source in current tree.'
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'vndk_version', type=int,
help='VNDK snapshot version to check, e.g. "27".')
return parser.parse_args()
def main():
"""For local testing purposes.
Note: VNDK snapshot must be already installed under
prebuilts/vndk/v{version}.
"""
ANDROID_BUILD_TOP = utils.get_android_build_top()
PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
'prebuilts/vndk')
args = get_args()
vndk_version = args.vndk_version
install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
if not os.path.isdir(install_dir):
raise ValueError(
'Please provide valid VNDK version. {} does not exist.'
.format(install_dir))
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
try:
license_checker.check_gpl_projects()
except ValueError as error:
print error
raise
if __name__ == '__main__':
main()

53
vndk/snapshot/gen_buildfiles.py
View File

@@ -18,6 +18,8 @@
import os
import sys
import utils
class GenBuildFile(object):
"""Generates Android.mk and Android.bp for prebuilts/vndk/v{version}."""
@@ -53,7 +55,7 @@ class GenBuildFile(object):
txt_filename: string, file name in VNDK snapshot
"""
prebuilt_list = []
txts = find(self._install_dir, [txt_filename])
txts = utils.find(self._install_dir, [txt_filename])
for txt in txts:
path_to_txt = os.path.join(self._install_dir, txt)
with open(path_to_txt, 'r') as f:
@@ -126,7 +128,7 @@ class GenBuildFile(object):
Args:
prebuilt: string, name of ETC prebuilt object
"""
etc_path = find(self._install_dir, prebuilt)[0]
etc_path = utils.find(self._install_dir, prebuilt)[0]
etc_sub_path = etc_path[etc_path.index('/') + 1:]
return (
@@ -202,7 +204,7 @@ class GenBuildFile(object):
notice = ''
notice_file_name = '{}.txt'.format(prebuilt)
notices_dir = os.path.join(self._install_dir, 'NOTICE_FILES')
notice_files = find(notices_dir, [notice_file_name])
notice_files = utils.find(notices_dir, [notice_file_name])
if len(notice_files) > 0:
notice = '{ind}notice: "{notice_file_path}",\n'.format(
ind=self.INDENT,
@@ -241,15 +243,13 @@ class GenBuildFile(object):
prebuilt: string, name of prebuilt object
"""
arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT)
src_paths = find(self._install_dir, [prebuilt])
# if len(src_paths) < 4:
# print prebuilt, src_paths
src_paths = utils.find(self._install_dir, [prebuilt])
for src in sorted(src_paths):
arch_srcs += ('{ind}{ind}{arch}: {{\n'
'{ind}{ind}{ind}srcs: ["{src}"],\n'
'{ind}{ind}}},\n'.format(
ind=self.INDENT,
arch=arch_from_path(src),
arch=utils.arch_from_path(src),
src=src))
arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
return arch_srcs
@@ -287,48 +287,15 @@ class GenBuildFile(object):
arch_srcs=arch_srcs))
def find(path, names):
"""Finds a list of files in a directory that match the given names.
Args:
path: string, absolute path of directory from which to find files
names: list of strings, names of the files to find
"""
found = []
for root, _, files in os.walk(path):
for file_name in sorted(files):
if file_name in names:
abspath = os.path.abspath(os.path.join(root, file_name))
rel_to_root = abspath.replace(os.path.abspath(path), '')
found.append(rel_to_root[1:]) # strip leading /
return found
def arch_from_path(path):
"""Extracts archfrom given VNDK snapshot path.
Args:
path: string, path relative to prebuilts/vndk/v{version}
Returns:
arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64")
"""
return path.split('/')[0].split('-')[1]
def main():
"""For local testing purposes.
Note: VNDK snapshot must be already installed under
prebuilts/vndk/v{version}.
"""
ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
if not ANDROID_BUILD_TOP:
print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
'\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
sys.exit(1)
PREBUILTS_VNDK_DIR = os.path.realpath(
os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk'))
ANDROID_BUILD_TOP = utils.get_android_build_top()
PREBUILTS_VNDK_DIR = utils.join_realpath(
ANDROID_BUILD_TOP, 'prebuilts/vndk')
vndk_version = 27 # set appropriately
install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))

53
vndk/snapshot/update.py
View File

@@ -26,24 +26,14 @@ import sys
import tempfile
import textwrap
import utils
from check_gpl_license import GPLChecker
from gen_buildfiles import GenBuildFile
ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
if not ANDROID_BUILD_TOP:
print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
'\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
sys.exit(1)
DIST_DIR = os.getenv('DIST_DIR')
if not DIST_DIR:
OUT_DIR = os.getenv('OUT_DIR')
if OUT_DIR:
DIST_DIR = os.path.realpath(os.path.join(OUT_DIR, 'dist'))
else:
DIST_DIR = os.path.realpath(os.path.join(ANDROID_BUILD_TOP, 'out/dist'))
PREBUILTS_VNDK_DIR = os.path.realpath(
os.path.join(ANDROID_BUILD_TOP, 'prebuilts/vndk'))
ANDROID_BUILD_TOP = utils.get_android_build_top()
DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP))
PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk')
def logger():
@@ -103,21 +93,23 @@ def install_snapshot(branch, build, install_dir):
artifact_dir = tempdir
os.chdir(tempdir)
logger().info('Fetching {pattern} from {branch} (bid: {build})'
logger().info(
'Fetching {pattern} from {branch} (bid: {build})'
.format(pattern=artifact_pattern, branch=branch, build=build))
fetch_artifact(branch, build, artifact_pattern)
manifest_pattern = 'manifest_{}.xml'.format(build)
manifest_name = 'manifest.xml'
logger().info('Fetching {file} from {branch} (bid: {build})'.format(
file=manifest_pattern, branch=branch, build=build))
logger().info(
'Fetching {file} from {branch} (bid: {build})'.format(
file=manifest_pattern, branch=branch, build=build))
fetch_artifact(branch, build, manifest_pattern, manifest_name)
shutil.move(manifest_name, install_dir)
os.chdir(install_dir)
else:
logger().info('Fetching local VNDK snapshot from {}'.format(
DIST_DIR))
logger().info(
'Fetching local VNDK snapshot from {}'.format(DIST_DIR))
artifact_dir = DIST_DIR
artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
@@ -148,9 +140,10 @@ def gather_notice_files():
notices_dir_per_arch = os.path.join(arch_dir, notices_dir_name)
if os.path.isdir(notices_dir_per_arch):
for notice_file in glob.glob(
'{}/*.txt'.format(notices_dir_per_arch)):
if not os.path.isfile(os.path.join(notices_dir_name,
os.path.basename(notice_file))):
'{}/*.txt'.format(notices_dir_per_arch)):
if not os.path.isfile(
os.path.join(notices_dir_name,
os.path.basename(notice_file))):
shutil.copy(notice_file, notices_dir_name)
shutil.rmtree(notices_dir_per_arch)
@@ -163,6 +156,14 @@ def update_buildfiles(buildfile_generator):
buildfile_generator.generate_android_bp()
def check_gpl_license(license_checker):
try:
license_checker.check_gpl_projects()
except ValueError as error:
print '***CANNOT INSTALL VNDK SNAPSHOT***', error
raise
def commit(branch, build, version):
logger().info('Making commit...')
check_call(['git', 'add', '.'])
@@ -186,7 +187,7 @@ def get_args():
help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of '
'Android Build server.'))
parser.add_argument(
'--use-current-branch',action='store_true',
'--use-current-branch', action='store_true',
help='Perform the update in the current branch. Do not repo start.')
parser.add_argument(
'-v', '--verbose', action='count', default=0,
@@ -239,6 +240,8 @@ def main():
update_buildfiles(buildfile_generator)
if not args.local:
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP)
check_gpl_license(license_checker)
commit(args.branch, args.build, vndk_version)

75
vndk/snapshot/utils.py Normal file
View File

@@ -0,0 +1,75 @@
#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Utility functions for VNDK snapshot."""
import os
import sys
def get_android_build_top():
ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
if not ANDROID_BUILD_TOP:
print('Error: Missing ANDROID_BUILD_TOP env variable. Please run '
'\'. build/envsetup.sh; lunch <build target>\'. Exiting script.')
sys.exit(1)
return ANDROID_BUILD_TOP
def join_realpath(root, *args):
return os.path.realpath(os.path.join(root, *args))
def _get_dir_from_env(env_var, default):
return os.path.realpath(os.getenv(env_var, default))
def get_out_dir(android_build_top):
return _get_dir_from_env('OUT_DIR', join_realpath(android_build_top,
'out'))
def get_dist_dir(out_dir):
return _get_dir_from_env('DIST_DIR', join_realpath(out_dir, 'dist'))
def arch_from_path(path):
"""Extracts arch from given VNDK snapshot path.
Args:
path: string, path relative to prebuilts/vndk/v{version}
Returns:
arch string, (e.g., "arm" or "arm64" or "x86" or "x86_64")
"""
return path.split('/')[0].split('-')[1]
def find(path, names):
"""Returns a list of files in a directory that match the given names.
Args:
path: string, absolute path of directory from which to find files
names: list of strings, names of the files to find
"""
found = []
for root, _, files in os.walk(path):
for file_name in sorted(files):
if file_name in names:
abspath = os.path.abspath(os.path.join(root, file_name))
rel_to_root = abspath.replace(os.path.abspath(path), '')
found.append(rel_to_root[1:]) # strip leading /
return found