Merge changes from topic "repo_diff"

* changes:
  Check project path to make sure it actully exists
  Add option --ignore_error_during_sync
  Write results to subfolders
  Rename exclusions.txt in the script

tools/repo_diff/repo_diff_android.py

@@ -10,6 +10,7 @@ This script:
 """
 
 import argparse
+import datetime
 import os
 import subprocess
 import repo_diff_trees
@@ -24,7 +25,7 @@ DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10"
 DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
 DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1"
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "exclusions.txt")
+DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt")
 
 
 def parse_args():
@@ -48,8 +49,12 @@ def parse_args():
                       help="exclusions file",
                       default=DEFAULT_EXCLUSIONS_FILE)
   parser.add_argument("-t", "--tag",
-                      help="release tag (optional). If not set then will"
+                      help="release tag (optional). If not set then will "
                       "sync the latest in the branch.")
+  parser.add_argument("-i", "--ignore_error_during_sync",
+                      action="store_true",
+                      help="repo sync might fail due to varios reasons. "
+                      "Ignore these errors and move on. Use with caution.")
 
   return parser.parse_args()
 
@@ -70,7 +75,7 @@ def repo_init(url, rev, workspace):
                           (url, rev), cwd=workspace, shell=True)
 
 
-def repo_sync(workspace, retry=5):
+def repo_sync(workspace, ignore_error, retry=5):
   """Repo sync."""
 
   count = 0
@@ -80,11 +85,15 @@ def repo_sync(workspace, retry=5):
           (count, retry, workspace))
 
     try:
-      subprocess.check_output(("repo sync --jobs=24 --current-branch --quiet "
+      command = "repo sync --jobs=24 --current-branch --quiet"
-                               "--no-tags --no-clone-bundle"),
+      command += " --no-tags --no-clone-bundle"
-                              cwd=workspace, shell=True)
+      if ignore_error:
+        command += " --force-broken"
+      subprocess.check_output(command, cwd=workspace, shell=True)
     except subprocess.CalledProcessError as e:
       print "Error: %s" % e.output
+      if count == retry and not ignore_error:
+        raise e
     # Stop retrying if the repo sync was successful
     else:
       break
@@ -107,7 +116,7 @@ def get_build_id(workspace):
                                  shell=True).rstrip()
 
 
-def repo_sync_specific_release(url, branch, tag, workspace):
+def repo_sync_specific_release(url, branch, tag, workspace, ignore_error):
   """Repo sync source with the specific release tag."""
 
   if not os.path.exists(workspace):
@@ -116,16 +125,19 @@ def repo_sync_specific_release(url, branch, tag, workspace):
   manifest_path = os.path.join(workspace, ".repo", "manifests")
 
   repo_init(url, branch, workspace)
+
   if tag:
     rev = get_commit_with_keyword(manifest_path, tag)
     if not rev:
       raise(ValueError("could not find a manifest revision for tag " + tag))
     repo_init(url, rev, workspace)
-  repo_sync(workspace)
+
+  repo_sync(workspace, ignore_error)
 
 
-def diff(manifest_url, manifest_branch, tag, upstream_manifest_url,
+def diff(manifest_url, manifest_branch, tag, 
-         upstream_manifest_branch, exclusions_file):
+         upstream_manifest_url, upstream_manifest_branch,
+         exclusions_file, ignore_error_during_sync):
   """Syncs and diffs an Android workspace against an upstream workspace."""
 
   workspace = os.path.abspath(DOWNSTREAM_WORKSPACE)
@@ -135,9 +147,11 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url,
       manifest_url,
       manifest_branch,
       tag,
-      workspace)
+      workspace,
+      ignore_error_during_sync)
 
   build_id = None
+
   if tag:
     # get the build_id so that we know which rev of upstream we need
     build_id = get_build_id(workspace)
@@ -149,14 +163,26 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url,
       upstream_manifest_url,
       upstream_manifest_branch,
       build_id,
-      upstream_workspace)
+      upstream_workspace,
+      ignore_error_during_sync)
+
+
+  # make output folder
+  if tag:
+    output_folder = os.path.abspath(tag.replace(" ", "_"))
+  else:
+    current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
+    output_folder = os.path.abspath(current_time)
+
+  if not os.path.exists(output_folder):
+      os.makedirs(output_folder)
 
   # do the comparison
   repo_diff_trees.diff(
       upstream_workspace,
       workspace,
-      os.path.abspath("project.csv"),
+      os.path.join(output_folder, "project.csv"),
-      os.path.abspath("commit.csv"),
+      os.path.join(output_folder, "commit.csv"),
       os.path.abspath(exclusions_file),
   )
 
@@ -164,9 +190,13 @@ def diff(manifest_url, manifest_branch, tag, upstream_manifest_url,
 def main():
   args = parse_args()
 
-  diff(args.manifest_url, args.manifest_branch, args.tag,
+  diff(args.manifest_url,
-       args.upstream_manifest_url, args.upstream_manifest_branch,
+       args.manifest_branch,
-       args.exclusions_file)
+       args.tag,
+       args.upstream_manifest_url,
+       args.upstream_manifest_branch,
+       args.exclusions_file,
+       args.ignore_error_during_sync)
 
 if __name__ == "__main__":
   main()

tools/repo_diff/repo_diff_trees.py

@@ -45,6 +45,11 @@ def get_projects(source_tree):
     path = project.get('path', project.get('name'))
     path = os.path.abspath(os.path.join(source_tree, path))
     name = project.get('name')
+
+    # check if project files actually exist
+    if not os.path.exists(path):
+      continue
+
     projects[name] = path
 
   return projects