Merge changes I98a86296,I2950b566,If51918fd

* changes:
  Allow user to specificy local_dir value
  Check parent commit if revision is from *-release
  Allow logging across modules
This commit is contained in:
Treehugger Robot
2018-06-20 02:29:08 +00:00
committed by Gerrit Code Review
4 changed files with 182 additions and 72 deletions

View File

@@ -17,6 +17,7 @@
import argparse import argparse
import glob import glob
import logging
import os import os
import shutil import shutil
import subprocess import subprocess
@@ -25,6 +26,8 @@ import xml.etree.ElementTree as xml_tree
import utils import utils
logger = utils.logger(__name__)
class GPLChecker(object): class GPLChecker(object):
"""Checks that all GPL projects in a VNDK snapshot have released sources. """Checks that all GPL projects in a VNDK snapshot have released sources.
@@ -57,7 +60,7 @@ class GPLChecker(object):
manifest_file=self._manifest_file)) manifest_file=self._manifest_file))
def _parse_module_paths(self): def _parse_module_paths(self):
"""Parses the module_path.txt files into a dictionary, """Parses the module_paths.txt files into a dictionary,
Returns: Returns:
module_paths: dict, e.g. {libfoo.so: some/path/here} module_paths: dict, e.g. {libfoo.so: some/path/here}
@@ -102,12 +105,57 @@ class GPLChecker(object):
git_project_path: string, path relative to ANDROID_BUILD_TOP git_project_path: string, path relative to ANDROID_BUILD_TOP
""" """
path = utils.join_realpath(self._android_build_top, git_project_path) path = utils.join_realpath(self._android_build_top, git_project_path)
try:
subprocess.check_call( def _check_rev_list(revision):
['git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)]) """Checks whether revision is reachable from HEAD of git project."""
logger.info('Checking if revision {rev} exists in {proj}'.format(
rev=revision, proj=git_project_path))
try:
cmd = [
'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)
]
output = utils.check_output(cmd, logger).strip()
except subprocess.CalledProcessError as error:
logger.error('Error: {}'.format(error))
return False
else:
if output:
logger.debug(
'{proj} does not have the following revisions: {rev}'.
format(proj=git_project_path, rev=output))
return False
else:
logger.info(
'Found revision {rev} in project {proj}'.format(
rev=revision, proj=git_project_path))
return True return True
except subprocess.CalledProcessError:
return False if not _check_rev_list(revision):
# VNDK snapshots built from a *-release branch will have merge
# CLs in the manifest because the *-dev branch is merged to the
# *-release branch periodically. In order to extract the
# revision relevant to the source of the git_project_path,
# we fetch the *-release branch and get the revision of the
# parent commit with FETCH_HEAD^2.
logger.info(
'Checking if the parent of revision {rev} exists in {proj}'.
format(rev=revision, proj=git_project_path))
try:
cmd = ['git', '-C', path, 'fetch', 'goog', revision]
utils.check_call(cmd, logger)
cmd = ['git', '-C', path, 'rev-parse', 'FETCH_HEAD^2']
parent_revision = utils.check_output(cmd, logger).strip()
except subprocess.CalledProcessError as error:
logger.error(
'Failed to get parent of revision {rev}: {err}'.format(
rev=revision, err=error))
raise
else:
if not _check_rev_list(parent_revision):
return False
return True
def check_gpl_projects(self): def check_gpl_projects(self):
"""Checks that all GPL projects have released sources. """Checks that all GPL projects have released sources.
@@ -115,7 +163,7 @@ class GPLChecker(object):
Raises: Raises:
ValueError: There are GPL projects with unreleased sources. ValueError: There are GPL projects with unreleased sources.
""" """
print 'Starting license check for GPL projects...' logger.info('Starting license check for GPL projects...')
notice_files = glob.glob('{}/*'.format(self._notice_files_dir)) notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
if len(notice_files) == 0: if len(notice_files) == 0:
@@ -132,10 +180,10 @@ class GPLChecker(object):
gpl_projects.append(lib_name) gpl_projects.append(lib_name)
if not gpl_projects: if not gpl_projects:
print 'No GPL projects found.' logger.info('No GPL projects found.')
return return
print 'GPL projects found:', ', '.join(gpl_projects) logger.info('GPL projects found: {}'.format(', '.join(gpl_projects)))
module_paths = self._parse_module_paths() module_paths = self._parse_module_paths()
manifest_projects = self._parse_manifest() manifest_projects = self._parse_manifest()
@@ -162,23 +210,30 @@ class GPLChecker(object):
format(lib=lib, module_paths=self.MODULE_PATHS_TXT)) format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
if released_projects: if released_projects:
print 'Released GPL projects:', released_projects logger.info('Released GPL projects: {}'.format(released_projects))
if unreleased_projects: if unreleased_projects:
raise ValueError( raise ValueError(
('FAIL: The following GPL projects have NOT been released in ' ('FAIL: The following GPL projects have NOT been released in '
'current tree: {}'.format(unreleased_projects))) 'current tree: {}'.format(unreleased_projects)))
print 'PASS: All GPL projects have source in current tree.' logger.info('PASS: All GPL projects have source in current tree.')
def get_args(): def get_args():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'vndk_version', type=int, 'vndk_version',
type=int,
help='VNDK snapshot version to check, e.g. "27".') help='VNDK snapshot version to check, e.g. "27".')
parser.add_argument('-b', '--branch', help='Branch to pull manifest from.') parser.add_argument('-b', '--branch', help='Branch to pull manifest from.')
parser.add_argument('--build', help='Build number to pull manifest from.') parser.add_argument('--build', help='Build number to pull manifest from.')
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase output verbosity, e.g. "-v", "-vv".')
return parser.parse_args() return parser.parse_args()
@@ -199,13 +254,14 @@ def main():
raise ValueError( raise ValueError(
'Please provide valid VNDK version. {} does not exist.' 'Please provide valid VNDK version. {} does not exist.'
.format(install_dir)) .format(install_dir))
utils.set_logging_config(args.verbose)
temp_artifact_dir = tempfile.mkdtemp() temp_artifact_dir = tempfile.mkdtemp()
os.chdir(temp_artifact_dir) os.chdir(temp_artifact_dir)
manifest_pattern = 'manifest_{}.xml'.format(args.build) manifest_pattern = 'manifest_{}.xml'.format(args.build)
print 'Fetching {file} from {branch} (bid: {build})'.format(
file=manifest_pattern, branch=args.branch, build=args.build)
manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME)
logger.info('Fetching {file} from {branch} (bid: {build})'.format(
file=manifest_pattern, branch=args.branch, build=args.build))
utils.fetch_artifact(args.branch, args.build, manifest_pattern, utils.fetch_artifact(args.branch, args.build, manifest_pattern,
manifest_dest) manifest_dest)
@@ -214,12 +270,14 @@ def main():
try: try:
license_checker.check_gpl_projects() license_checker.check_gpl_projects()
except ValueError as error: except ValueError as error:
print error logger.error('Error: {}'.format(error))
raise raise
finally: finally:
print 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir) logger.info('Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
shutil.rmtree(temp_artifact_dir) shutil.rmtree(temp_artifact_dir)
logger.info('Done.')
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@@ -15,12 +15,16 @@
# limitations under the License. # limitations under the License.
# #
import argparse
import glob import glob
import logging
import os import os
import sys import sys
import utils import utils
logger = utils.logger(__name__)
class GenBuildFile(object): class GenBuildFile(object):
"""Generates Android.mk and Android.bp for VNDK snapshot. """Generates Android.mk and Android.bp for VNDK snapshot.
@@ -342,6 +346,21 @@ class GenBuildFile(object):
arch_srcs=arch_srcs)) arch_srcs=arch_srcs))
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'vndk_version',
type=int,
help='VNDK snapshot version to install, e.g. "27".')
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase output verbosity, e.g. "-v", "-vv".')
return parser.parse_args()
def main(): def main():
"""For local testing purposes. """For local testing purposes.
@@ -352,13 +371,21 @@ def main():
PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
'prebuilts/vndk') 'prebuilts/vndk')
vndk_version = 27 # set appropriately args = get_args()
vndk_version = args.vndk_version
install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
if not os.path.isdir(install_dir):
raise ValueError(
'Please provide valid VNDK version. {} does not exist.'
.format(install_dir))
utils.set_logging_config(args.verbose)
buildfile_generator = GenBuildFile(install_dir, vndk_version) buildfile_generator = GenBuildFile(install_dir, vndk_version)
buildfile_generator.generate_android_mk() buildfile_generator.generate_android_mk()
buildfile_generator.generate_android_bp() buildfile_generator.generate_android_bp()
logger.info('Done.')
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@@ -33,28 +33,20 @@ from check_gpl_license import GPLChecker
from gen_buildfiles import GenBuildFile from gen_buildfiles import GenBuildFile
ANDROID_BUILD_TOP = utils.get_android_build_top() ANDROID_BUILD_TOP = utils.get_android_build_top()
DIST_DIR = utils.get_dist_dir(utils.get_out_dir(ANDROID_BUILD_TOP))
PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk')
logger = utils.logger(__name__)
def logger():
return logging.getLogger(__name__)
def check_call(cmd):
logger().debug('Running `{}`'.format(' '.join(cmd)))
subprocess.check_call(cmd)
def start_branch(build): def start_branch(build):
branch_name = 'update-' + (build or 'local') branch_name = 'update-' + (build or 'local')
logger().info('Creating branch {branch} in {dir}'.format( logger.info('Creating branch {branch} in {dir}'.format(
branch=branch_name, dir=os.getcwd())) branch=branch_name, dir=os.getcwd()))
check_call(['repo', 'start', branch_name, '.']) utils.check_call(['repo', 'start', branch_name, '.'], logger)
def remove_old_snapshot(install_dir): def remove_old_snapshot(install_dir):
logger().info('Removing any old files in {}'.format(install_dir)) logger.info('Removing any old files in {}'.format(install_dir))
for file in glob.glob('{}/*'.format(install_dir)): for file in glob.glob('{}/*'.format(install_dir)):
try: try:
if os.path.isfile(file): if os.path.isfile(file):
@@ -62,19 +54,20 @@ def remove_old_snapshot(install_dir):
elif os.path.isdir(file): elif os.path.isdir(file):
shutil.rmtree(file) shutil.rmtree(file)
except Exception as error: except Exception as error:
print error logger.error('Error: {}'.format(error))
sys.exit(1) sys.exit(1)
def install_snapshot(branch, build, install_dir, temp_artifact_dir): def install_snapshot(branch, build, local_dir, install_dir, temp_artifact_dir):
"""Installs VNDK snapshot build artifacts to prebuilts/vndk/v{version}. """Installs VNDK snapshot build artifacts to prebuilts/vndk/v{version}.
1) Fetch build artifacts from Android Build server or from local DIST_DIR 1) Fetch build artifacts from Android Build server or from local_dir
2) Unzip build artifacts 2) Unzip build artifacts
Args: Args:
branch: string or None, branch name of build artifacts branch: string or None, branch name of build artifacts
build: string or None, build number of build artifacts build: string or None, build number of build artifacts
local_dir: string or None, local dir to pull artifacts from
install_dir: string, directory to install VNDK snapshot install_dir: string, directory to install VNDK snapshot
temp_artifact_dir: string, temp directory to hold build artifacts fetched temp_artifact_dir: string, temp directory to hold build artifacts fetched
from Android Build server. For 'local' option, is set to None. from Android Build server. For 'local' option, is set to None.
@@ -84,37 +77,33 @@ def install_snapshot(branch, build, install_dir, temp_artifact_dir):
if branch and build: if branch and build:
artifact_dir = temp_artifact_dir artifact_dir = temp_artifact_dir
os.chdir(temp_artifact_dir) os.chdir(temp_artifact_dir)
logger().info('Fetching {pattern} from {branch} (bid: {build})'.format( logger.info('Fetching {pattern} from {branch} (bid: {build})'.format(
pattern=artifact_pattern, branch=branch, build=build)) pattern=artifact_pattern, branch=branch, build=build))
utils.fetch_artifact(branch, build, artifact_pattern) utils.fetch_artifact(branch, build, artifact_pattern)
manifest_pattern = 'manifest_{}.xml'.format(build) manifest_pattern = 'manifest_{}.xml'.format(build)
logger().info('Fetching {file} from {branch} (bid: {build})'.format( logger.info('Fetching {file} from {branch} (bid: {build})'.format(
file=manifest_pattern, branch=branch, build=build)) file=manifest_pattern, branch=branch, build=build))
utils.fetch_artifact(branch, build, manifest_pattern, utils.fetch_artifact(branch, build, manifest_pattern,
utils.MANIFEST_FILE_NAME) utils.MANIFEST_FILE_NAME)
os.chdir(install_dir) os.chdir(install_dir)
else: elif local_dir:
logger().info('Fetching local VNDK snapshot from {}'.format(DIST_DIR)) logger.info('Fetching local VNDK snapshot from {}'.format(local_dir))
artifact_dir = DIST_DIR artifact_dir = local_dir
artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern)) artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
artifact_cnt = len(artifacts)
if artifact_cnt < 4:
raise RuntimeError(
'Expected four android-vndk-*.zip files in {path}. Instead '
'found {cnt}.'.format(path=artifact_dir, cnt=artifact_cnt))
for artifact in artifacts: for artifact in artifacts:
logger().info('Unzipping VNDK snapshot: {}'.format(artifact)) logger.info('Unzipping VNDK snapshot: {}'.format(artifact))
check_call(['unzip', '-q', artifact, '-d', install_dir]) utils.check_call(['unzip', '-qn', artifact, '-d', install_dir], logger)
def gather_notice_files(install_dir): def gather_notice_files(install_dir):
"""Gathers all NOTICE files to a common NOTICE_FILES directory.""" """Gathers all NOTICE files to a common NOTICE_FILES directory."""
common_notices_dir = utils.NOTICE_FILES_DIR_PATH common_notices_dir = utils.NOTICE_FILES_DIR_PATH
logger().info('Creating {} directory...'.format(common_notices_dir)) logger.info('Creating {} directory to gather all NOTICE files...'.format(
common_notices_dir))
os.makedirs(common_notices_dir) os.makedirs(common_notices_dir)
for variant in utils.get_snapshot_variants(install_dir): for variant in utils.get_snapshot_variants(install_dir):
notices_dir_per_variant = os.path.join(variant, notices_dir_per_variant = os.path.join(variant,
@@ -138,6 +127,7 @@ def revise_ld_config_txt_if_needed(vndk_version):
Args: Args:
vndk_version: string, version of VNDK snapshot vndk_version: string, version of VNDK snapshot
""" """
logger.info('Revising ld.config.txt for O-MR1...')
if vndk_version == '27': if vndk_version == '27':
re_pattern = '(system\/\${LIB}\/vndk(?:-sp)?)([:/]|$)' re_pattern = '(system\/\${LIB}\/vndk(?:-sp)?)([:/]|$)'
VNDK_INSTALL_DIR_RE = re.compile(re_pattern, flags=re.MULTILINE) VNDK_INSTALL_DIR_RE = re.compile(re_pattern, flags=re.MULTILINE)
@@ -151,10 +141,10 @@ def revise_ld_config_txt_if_needed(vndk_version):
def update_buildfiles(buildfile_generator): def update_buildfiles(buildfile_generator):
logger().info('Generating Android.mk file...') logger.info('Generating Android.mk file...')
buildfile_generator.generate_android_mk() buildfile_generator.generate_android_mk()
logger().info('Generating Android.bp files...') logger.info('Generating Android.bp files...')
buildfile_generator.generate_android_bp() buildfile_generator.generate_android_bp()
@@ -162,37 +152,43 @@ def check_gpl_license(license_checker):
try: try:
license_checker.check_gpl_projects() license_checker.check_gpl_projects()
except ValueError as error: except ValueError as error:
print '***CANNOT INSTALL VNDK SNAPSHOT***', error logger.error('***CANNOT INSTALL VNDK SNAPSHOT***: {}'.format(error))
raise raise
def commit(branch, build, version): def commit(branch, build, version):
logger().info('Making commit...') logger.info('Making commit...')
check_call(['git', 'add', '.']) utils.check_call(['git', 'add', '.'], logger)
message = textwrap.dedent("""\ message = textwrap.dedent("""\
Update VNDK snapshot v{version} to build {build}. Update VNDK snapshot v{version} to build {build}.
Taken from branch {branch}.""").format( Taken from branch {branch}.""").format(
version=version, branch=branch, build=build) version=version, branch=branch, build=build)
check_call(['git', 'commit', '-m', message]) utils.check_call(['git', 'commit', '-m', message], logger)
def get_args(): def get_args():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'vndk_version', type=int, 'vndk_version',
type=int,
help='VNDK snapshot version to install, e.g. "27".') help='VNDK snapshot version to install, e.g. "27".')
parser.add_argument('-b', '--branch', help='Branch to pull build from.') parser.add_argument('-b', '--branch', help='Branch to pull build from.')
parser.add_argument('--build', help='Build number to pull.') parser.add_argument('--build', help='Build number to pull.')
parser.add_argument( parser.add_argument(
'--local', action='store_true', '--local',
help=('Fetch local VNDK snapshot artifacts from DIST_DIR instead of ' help=('Fetch local VNDK snapshot artifacts from specified local '
'Android Build server.')) 'directory instead of Android Build server. '
'Example: --local=/path/to/local/dir'))
parser.add_argument( parser.add_argument(
'--use-current-branch', action='store_true', '--use-current-branch',
action='store_true',
help='Perform the update in the current branch. Do not repo start.') help='Perform the update in the current branch. Do not repo start.')
parser.add_argument( parser.add_argument(
'-v', '--verbose', action='count', default=0, '-v',
'--verbose',
action='count',
default=0,
help='Increase output verbosity, e.g. "-v", "-vv".') help='Increase output verbosity, e.g. "-v", "-vv".')
return parser.parse_args() return parser.parse_args()
@@ -206,10 +202,10 @@ def main():
raise ValueError( raise ValueError(
'When --local option is set, --branch or --build cannot be ' 'When --local option is set, --branch or --build cannot be '
'specified.') 'specified.')
elif not os.path.isdir(DIST_DIR): elif not os.path.isdir(args.local):
raise RuntimeError( raise RuntimeError(
'The --local option is set, but DIST_DIR={} does not exist.'. 'The specified local directory, {}, does not exist.'.format(
format(DIST_DIR)) args.local))
else: else:
if not (args.build and args.branch): if not (args.build and args.branch):
raise ValueError( raise ValueError(
@@ -225,9 +221,7 @@ def main():
'Please request a new git project for prebuilts/vndk/v{ver} ' 'Please request a new git project for prebuilts/vndk/v{ver} '
'before installing new snapshot.'.format(ver=vndk_version)) 'before installing new snapshot.'.format(ver=vndk_version))
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) utils.set_logging_config(args.verbose)
verbosity = min(args.verbose, 2)
logging.basicConfig(level=verbose_map[verbosity])
os.chdir(install_dir) os.chdir(install_dir)
@@ -241,9 +235,8 @@ def main():
if not args.local: if not args.local:
temp_artifact_dir = tempfile.mkdtemp() temp_artifact_dir = tempfile.mkdtemp()
install_status = True
try: try:
install_snapshot(args.branch, args.build, install_dir, install_snapshot(args.branch, args.build, args.local, install_dir,
temp_artifact_dir) temp_artifact_dir)
gather_notice_files(install_dir) gather_notice_files(install_dir)
revise_ld_config_txt_if_needed(vndk_version) revise_ld_config_txt_if_needed(vndk_version)
@@ -255,17 +248,23 @@ def main():
license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
temp_artifact_dir) temp_artifact_dir)
check_gpl_license(license_checker) check_gpl_license(license_checker)
except: logger.info(
logger().info('FAILED TO INSTALL SNAPSHOT') 'Successfully updated VNDK snapshot v{}'.format(vndk_version))
install_status = False except Exception as error:
logger.error('FAILED TO INSTALL SNAPSHOT: {}'.format(error))
raise
finally: finally:
if temp_artifact_dir: if temp_artifact_dir:
logger().info( logger.info(
'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
shutil.rmtree(temp_artifact_dir) shutil.rmtree(temp_artifact_dir)
if not args.local and install_status: if not args.local:
commit(args.branch, args.build, vndk_version) commit(args.branch, args.build, vndk_version)
logger.info('Successfully created commit for VNDK snapshot v{}'.format(
vndk_version))
logger.info('Done.')
if __name__ == '__main__': if __name__ == '__main__':

View File

@@ -17,6 +17,7 @@
"""Utility functions for VNDK snapshot.""" """Utility functions for VNDK snapshot."""
import glob import glob
import logging
import os import os
import re import re
import subprocess import subprocess
@@ -34,6 +35,32 @@ NOTICE_FILES_DIR_NAME = 'NOTICE_FILES'
NOTICE_FILES_DIR_PATH = os.path.join(COMMON_DIR_PATH, NOTICE_FILES_DIR_NAME) NOTICE_FILES_DIR_PATH = os.path.join(COMMON_DIR_PATH, NOTICE_FILES_DIR_NAME)
def logger(name):
return logging.getLogger(name)
def set_logging_config(verbose_level):
verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
verbosity = min(verbose_level, 2)
logging.basicConfig(
format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
level=verbose_map[verbosity])
def check_call(cmd, logger=None):
logger = logger or logging
logger.debug('Running `{}`'.format(' '.join(cmd)))
subprocess.check_call(cmd)
def check_output(cmd, logger=None):
logger = logger or logging
logger.debug('Running `{}`'.format(' '.join(cmd)))
output = subprocess.check_output(cmd)
logger.debug('Output: `{}`'.format(output))
return output
def get_android_build_top(): def get_android_build_top():
ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP') ANDROID_BUILD_TOP = os.getenv('ANDROID_BUILD_TOP')
if not ANDROID_BUILD_TOP: if not ANDROID_BUILD_TOP:
@@ -129,5 +156,4 @@ def fetch_artifact(branch, build, pattern, destination='.'):
fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid', fetch_artifact_path, '--branch', branch, '--target=vndk', '--bid',
build, pattern, destination build, pattern, destination
] ]
print 'Running `{}`'.format(' '.join(cmd)) check_call(cmd, logger(__name__))
subprocess.check_call(cmd)