From fb4ba49c2099f26dc17d6d4702d0cd7a675d17e8 Mon Sep 17 00:00:00 2001 From: Po-Chien Hsueh Date: Thu, 28 Sep 2017 16:52:35 +0800 Subject: [PATCH 1/4] Rename exclusions.txt in the script The file name has been changed in a previous CL Test: run the script Change-Id: I7cc1b183989ae674e14e65321e09c51d5c689ad0 --- tools/repo_diff/repo_diff_android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/repo_diff/repo_diff_android.py b/tools/repo_diff/repo_diff_android.py index f47e71f0f..2cf5fb548 100644 --- a/tools/repo_diff/repo_diff_android.py +++ b/tools/repo_diff/repo_diff_android.py @@ -24,7 +24,7 @@ DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10" DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest" DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1" SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "exclusions.txt") +DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt") def parse_args(): From f9b8f0cce0dc6d016ca5a8ae597dfd82adf31f6d Mon Sep 17 00:00:00 2001 From: Po-Chien Hsueh Date: Thu, 28 Sep 2017 16:00:42 +0800 Subject: [PATCH 2/4] Write results to subfolders We are using this tools on multiple releases (tags). Create subfolders using release names, so that results won't be overwitten. Test: run the script against multiple releases Change-Id: I9a94940d630874a5b378431f20a6c1182cf11509 --- tools/repo_diff/repo_diff_android.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/repo_diff/repo_diff_android.py b/tools/repo_diff/repo_diff_android.py index 2cf5fb548..bd42520e9 100644 --- a/tools/repo_diff/repo_diff_android.py +++ b/tools/repo_diff/repo_diff_android.py @@ -10,6 +10,7 @@ This script: """ import argparse +import datetime import os import subprocess import repo_diff_trees @@ -116,11 +117,13 @@ def repo_sync_specific_release(url, branch, tag, workspace): manifest_path = os.path.join(workspace, ".repo", "manifests") repo_init(url, branch, workspace) + if tag: rev = get_commit_with_keyword(manifest_path, tag) if not rev: raise(ValueError("could not find a manifest revision for tag " + tag)) repo_init(url, rev, workspace) + repo_sync(workspace) @@ -138,6 +141,7 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, workspace) build_id = None + if tag: # get the build_id so that we know which rev of upstream we need build_id = get_build_id(workspace) @@ -151,12 +155,23 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, build_id, upstream_workspace) + + # make output folder + if tag: + output_folder = os.path.abspath(tag.replace(" ", "_")) + else: + current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S') + output_folder = os.path.abspath(current_time) + + if not os.path.exists(output_folder): + os.makedirs(output_folder) + # do the comparison repo_diff_trees.diff( upstream_workspace, workspace, - os.path.abspath("project.csv"), - os.path.abspath("commit.csv"), + os.path.join(output_folder, "project.csv"), + os.path.join(output_folder, "commit.csv"), os.path.abspath(exclusions_file), ) From 95ca7ccb8464f5b15643b053b372fb7337860237 Mon Sep 17 00:00:00 2001 From: Po-Chien Hsueh Date: Thu, 28 Sep 2017 16:09:56 +0800 Subject: [PATCH 3/4] Add option --ignore_error_during_sync Some old SHA1 might be invalid when we use this tool. Ignore these projects and keep going on. Test: run the script against an old release Change-Id: I5c89911759de6d122052e841eef0b016fa8b1422 --- tools/repo_diff/repo_diff_android.py | 43 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) mode change 100644 => 100755 tools/repo_diff/repo_diff_android.py diff --git a/tools/repo_diff/repo_diff_android.py b/tools/repo_diff/repo_diff_android.py old mode 100644 new mode 100755 index bd42520e9..822abb500 --- a/tools/repo_diff/repo_diff_android.py +++ b/tools/repo_diff/repo_diff_android.py @@ -49,8 +49,12 @@ def parse_args(): help="exclusions file", default=DEFAULT_EXCLUSIONS_FILE) parser.add_argument("-t", "--tag", - help="release tag (optional). If not set then will" + help="release tag (optional). If not set then will " "sync the latest in the branch.") + parser.add_argument("-i", "--ignore_error_during_sync", + action="store_true", + help="repo sync might fail due to varios reasons. " + "Ignore these errors and move on. Use with caution.") return parser.parse_args() @@ -71,7 +75,7 @@ def repo_init(url, rev, workspace): (url, rev), cwd=workspace, shell=True) -def repo_sync(workspace, retry=5): +def repo_sync(workspace, ignore_error, retry=5): """Repo sync.""" count = 0 @@ -81,11 +85,15 @@ def repo_sync(workspace, retry=5): (count, retry, workspace)) try: - subprocess.check_output(("repo sync --jobs=24 --current-branch --quiet " - "--no-tags --no-clone-bundle"), - cwd=workspace, shell=True) + command = "repo sync --jobs=24 --current-branch --quiet" + command += " --no-tags --no-clone-bundle" + if ignore_error: + command += " --force-broken" + subprocess.check_output(command, cwd=workspace, shell=True) except subprocess.CalledProcessError as e: print "Error: %s" % e.output + if count == retry and not ignore_error: + raise e # Stop retrying if the repo sync was successful else: break @@ -108,7 +116,7 @@ def get_build_id(workspace): shell=True).rstrip() -def repo_sync_specific_release(url, branch, tag, workspace): +def repo_sync_specific_release(url, branch, tag, workspace, ignore_error): """Repo sync source with the specific release tag.""" if not os.path.exists(workspace): @@ -124,11 +132,12 @@ def repo_sync_specific_release(url, branch, tag, workspace): raise(ValueError("could not find a manifest revision for tag " + tag)) repo_init(url, rev, workspace) - repo_sync(workspace) + repo_sync(workspace, ignore_error) -def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, - upstream_manifest_branch, exclusions_file): +def diff(manifest_url, manifest_branch, tag, + upstream_manifest_url, upstream_manifest_branch, + exclusions_file, ignore_error_during_sync): """Syncs and diffs an Android workspace against an upstream workspace.""" workspace = os.path.abspath(DOWNSTREAM_WORKSPACE) @@ -138,7 +147,8 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, manifest_url, manifest_branch, tag, - workspace) + workspace, + ignore_error_during_sync) build_id = None @@ -153,7 +163,8 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, upstream_manifest_url, upstream_manifest_branch, build_id, - upstream_workspace) + upstream_workspace, + ignore_error_during_sync) # make output folder @@ -179,9 +190,13 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url, def main(): args = parse_args() - diff(args.manifest_url, args.manifest_branch, args.tag, - args.upstream_manifest_url, args.upstream_manifest_branch, - args.exclusions_file) + diff(args.manifest_url, + args.manifest_branch, + args.tag, + args.upstream_manifest_url, + args.upstream_manifest_branch, + args.exclusions_file, + args.ignore_error_during_sync) if __name__ == "__main__": main() From 79a7b0e8f456960abec89250e614bebec9de9f09 Mon Sep 17 00:00:00 2001 From: Po-Chien Hsueh Date: Thu, 28 Sep 2017 17:15:08 +0800 Subject: [PATCH 4/4] Check project path to make sure it actully exists Some old projects might be removed for some reason, especially the downstream ones. It should be fine to just skip analysing them because people have stopped working on them. Test: run the script against an old release Change-Id: I563905565c4c502036159fce6a386bba13ba25ea --- tools/repo_diff/repo_diff_trees.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/repo_diff/repo_diff_trees.py b/tools/repo_diff/repo_diff_trees.py index d6e3691e2..cced73241 100644 --- a/tools/repo_diff/repo_diff_trees.py +++ b/tools/repo_diff/repo_diff_trees.py @@ -45,6 +45,11 @@ def get_projects(source_tree): path = project.get('path', project.get('name')) path = os.path.abspath(os.path.join(source_tree, path)) name = project.get('name') + + # check if project files actually exist + if not os.path.exists(path): + continue + projects[name] = path return projects