From 3f67c28b5203864a8f20d0a1ef5a47bd66a6f9ad Mon Sep 17 00:00:00 2001 From: Logan Chien Date: Wed, 6 Jun 2018 18:16:31 +0800 Subject: [PATCH] repo-pull: Check Gerrit Code Review domain name This commit adds a search on Gerrit Code Review URL so that a better error message can be shown to users and the setup instructions can be presented. Test: repo_pull.py json -g [gerrit] [query] Test: repo_review.py -g [gerrit] [query] -l Code-Review 2 Change-Id: I1f819c51e34c573f526ca3e30e237cdaf8a27160 --- tools/repo_pull/gerrit.py | 191 +++++++++++++++++++++++++++++++++ tools/repo_pull/repo_pull.py | 64 +---------- tools/repo_pull/repo_review.py | 114 ++------------------ 3 files changed, 203 insertions(+), 166 deletions(-) create mode 100755 tools/repo_pull/gerrit.py diff --git a/tools/repo_pull/gerrit.py b/tools/repo_pull/gerrit.py new file mode 100755 index 000000000..2ac2e14bc --- /dev/null +++ b/tools/repo_pull/gerrit.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function + +import argparse +import json +import os +import sys + +try: + from urllib.request import HTTPBasicAuthHandler, build_opener # PY3 +except ImportError: + from urllib2 import HTTPBasicAuthHandler, build_opener # PY2 + +try: + from urllib.parse import urlencode, urlparse # PY3 +except ImportError: + from urllib import urlencode # PY2 + from urlparse import urlparse # PY2 + + +def load_auth_credentials_from_file(cookie_file): + """Load credentials from an opened .gitcookies file.""" + credentials = {} + for lineno, line in enumerate(cookie_file, start=1): + if line.startswith('#HttpOnly_'): + line = line[len('#HttpOnly_'):] + + if not line or line[0] == '#': + continue + + row = line.split('\t') + if len(row) != 7: + continue + + domain = row[0] + cookie = row[6] + + sep = cookie.find('=') + if sep == -1: + continue + username = cookie[0:sep] + password = cookie[sep + 1:] + + credentials[domain] = (username, password) + return credentials + + +def load_auth_credentials(cookie_file_path): + """Load credentials from a .gitcookies file path.""" + with open(cookie_file_path, 'r') as cookie_file: + return load_auth_credentials_from_file(cookie_file) + + +def create_url_opener(cookie_file_path, domain): + """Load username and password from .gitcookies and return a URL opener with + an authentication handler.""" + + # Load authentication credentials + credentials = load_auth_credentials(cookie_file_path) + username, password = credentials[domain] + + # Create URL opener with authentication handler + auth_handler = HTTPBasicAuthHandler() + auth_handler.add_password(domain, domain, username, password) + return build_opener(auth_handler) + + +def create_url_opener_from_args(args): + """Create URL opener from command line arguments.""" + + domain = urlparse(args.gerrit).netloc + + try: + return create_url_opener(args.gitcookies, domain) + except KeyError: + print('error: Cannot find the domain "{}" in "{}". ' + .format(domain, args.gitcookies), file=sys.stderr) + print('error: Please check the Gerrit Code Review URL or follow the ' + 'instructions in ' + 'https://android.googlesource.com/platform/development/' + '+/master/tools/repo_pull#installation', file=sys.stderr) + sys.exit(1) + + +def _decode_xssi_json(data): + """Trim XSSI protector and decode JSON objects.""" + + # Decode UTF-8 + data = data.decode('utf-8') + + # Trim cross site script inclusion (XSSI) protector + if data[0:4] != ')]}\'': + raise ValueError('unexpected responsed content: ' + data) + data = data[4:] + + # Parse JSON objects + return json.loads(data) + + +def query_change_lists(url_opener, gerrit, query_string, limits): + """Query change lists.""" + data = [ + ('q', query_string), + ('o', 'CURRENT_REVISION'), + ('o', 'CURRENT_COMMIT'), + ('n', str(limits)), + ] + url = gerrit + '/a/changes/?' + urlencode(data) + + response_file = url_opener.open(url) + try: + return _decode_xssi_json(response_file.read()) + finally: + response_file.close() + + +def set_review(url_opener, gerrit_url, change_id, labels, message): + """Set review votes to a change list.""" + + url = '{}/a/changes/{}/revisions/current/review'.format( + gerrit_url, change_id) + + data = {} + if labels: + data['labels'] = labels + if message: + data['message'] = message + data = json.dumps(data).encode('utf-8') + + headers = { + 'Content-Type': 'application/json; charset=UTF-8', + } + + request = Request(url, data, headers) + response_file = url_opener.open(request) + try: + res_code = response_file.getcode() + res_json = _decode_xssi_json(response_file.read()) + return (res_code, res_json) + finally: + response_file.close() + + +def _parse_args(): + """Parse command line options.""" + parser = argparse.ArgumentParser() + + parser.add_argument('query', help='Change list query string') + parser.add_argument('-g', '--gerrit', required=True, + help='Gerrit review URL') + + parser.add_argument('--gitcookies', + default=os.path.expanduser('~/.gitcookies'), + help='Gerrit cookie file') + parser.add_argument('--limits', default=1000, + help='Max number of change lists') + + return parser.parse_args() + + +def main(): + args = _parse_args() + + # Query change lists + url_opener = create_url_opener_from_args(args) + change_lists = query_change_lists( + url_opener, args.gerrit, args.query, args.limits) + + # Print the result + json.dump(change_lists, sys.stdout, indent=4, separators=(', ', ': ')) + print() # Print the end-of-line + +if __name__ == '__main__': + main() diff --git a/tools/repo_pull/repo_pull.py b/tools/repo_pull/repo_pull.py index 3c40da644..c47f1bae7 100755 --- a/tools/repo_pull/repo_pull.py +++ b/tools/repo_pull/repo_pull.py @@ -20,6 +20,8 @@ from __future__ import print_function +from gerrit import create_url_opener_from_args, query_change_lists + import argparse import collections import itertools @@ -30,16 +32,6 @@ import re import sys import xml.dom.minidom -try: - from urllib.parse import urlencode # PY3 -except ImportError: - from urllib import urlencode # PY2 - -try: - from urllib.request import HTTPBasicAuthHandler, build_opener # PY3 -except ImportError: - from urllib2 import HTTPBasicAuthHandler, build_opener # PY2 - try: from __builtin__ import raw_input as input # PY2 except ImportError: @@ -167,54 +159,6 @@ def build_project_name_to_directory_dict(manifest_path): return project_dirs -def load_auth(cookie_file_path): - """Load username and password from .gitcookies and return an - HTTPBasicAuthHandler.""" - auth_handler = HTTPBasicAuthHandler() - with open(cookie_file_path, 'r') as cookie_file: - for lineno, line in enumerate(cookie_file, start=1): - if line.startswith('#HttpOnly_'): - line = line[len('#HttpOnly_'):] - if not line or line[0] == '#': - continue - row = line.split('\t') - if len(row) != 7: - continue - domain = row[0] - cookie = row[6] - sep = cookie.find('=') - if sep == -1: - continue - username = cookie[0:sep] - password = cookie[sep + 1:] - auth_handler.add_password(domain, domain, username, password) - return auth_handler - - -def query_change_lists(gerrit, query_string, gitcookies, limits): - """Query change lists.""" - data = [ - ('q', query_string), - ('o', 'CURRENT_REVISION'), - ('o', 'CURRENT_COMMIT'), - ('n', str(limits)), - ] - url = gerrit + '/a/changes/?' + urlencode(data) - - auth_handler = load_auth(gitcookies) - opener = build_opener(auth_handler) - - response_file = opener.open(url) - try: - # Trim cross site script inclusion (XSSI) protector - data = response_file.read().decode('utf-8')[4:] - - # Parse responsed JSON - return json.loads(data) - finally: - response_file.close() - - def group_and_sort_change_lists(change_lists, project_dirs): """Build a dict that maps projects to a list of topologically sorted change lists.""" @@ -453,8 +397,8 @@ def _get_manifest_xml_from_args(args): def _get_change_lists_from_args(args): """Query the change lists by args.""" - return query_change_lists(args.gerrit, args.query, args.gitcookies, - args.limits) + url_opener = create_url_opener_from_args(args) + return query_change_lists(url_opener, args.gerrit, args.query, args.limits) def _get_local_branch_name_from_args(args): diff --git a/tools/repo_pull/repo_review.py b/tools/repo_pull/repo_review.py index 9a404c818..bae2853f6 100755 --- a/tools/repo_pull/repo_review.py +++ b/tools/repo_pull/repo_review.py @@ -20,113 +20,12 @@ from __future__ import print_function +from gerrit import create_url_opener_from_args, query_change_lists, set_review + import argparse -import collections -import itertools import json -import multiprocessing import os -import re import sys -import xml.dom.minidom - -try: - from urllib.parse import urlencode # PY3 -except ImportError: - from urllib import urlencode # PY2 - -try: - from urllib.request import ( - HTTPBasicAuthHandler, Request, build_opener) # PY3 -except ImportError: - from urllib2 import HTTPBasicAuthHandler, Request, build_opener # PY2 - - -def load_auth(cookie_file_path): - """Load username and password from .gitcookies and return an - HTTPBasicAuthHandler.""" - auth_handler = HTTPBasicAuthHandler() - with open(cookie_file_path, 'r') as cookie_file: - for lineno, line in enumerate(cookie_file, start=1): - if line.startswith('#HttpOnly_'): - line = line[len('#HttpOnly_'):] - if not line or line[0] == '#': - continue - row = line.split('\t') - if len(row) != 7: - continue - domain = row[0] - cookie = row[6] - sep = cookie.find('=') - if sep == -1: - continue - username = cookie[0:sep] - password = cookie[sep + 1:] - auth_handler.add_password(domain, domain, username, password) - return auth_handler - - -def _decode_xssi_json(data): - """Trim XSSI protector and decode JSON objects.""" - # Trim cross site script inclusion (XSSI) protector - data = data.decode('utf-8')[4:] - # Parse JSON objects - return json.loads(data) - - -def query_change_lists(gerrit, query_string, gitcookies, limits): - """Query change lists.""" - data = [ - ('q', query_string), - ('o', 'CURRENT_REVISION'), - ('o', 'CURRENT_COMMIT'), - ('n', str(limits)), - ] - url = gerrit + '/a/changes/?' + urlencode(data) - - auth_handler = load_auth(gitcookies) - opener = build_opener(auth_handler) - - response_file = opener.open(url) - try: - return _decode_xssi_json(response_file.read()) - finally: - response_file.close() - - -def set_review(gerrit, gitcookies, change_id, labels, message): - """Set review votes to a change list.""" - - url = '{}/a/changes/{}/revisions/current/review'.format(gerrit, change_id) - - auth_handler = load_auth(gitcookies) - opener = build_opener(auth_handler) - - data = {} - if labels: - data['labels'] = labels - if message: - data['message'] = message - data = json.dumps(data).encode('utf-8') - - headers = { - 'Content-Type': 'application/json; charset=UTF-8', - } - - request = Request(url, data, headers) - response_file = opener.open(request) - try: - res_code = response_file.getcode() - res_json = _decode_xssi_json(response_file.read()) - return (res_code, res_json) - finally: - response_file.close() - - -def _get_change_lists_from_args(args): - """Query the change lists by args.""" - return query_change_lists(args.gerrit, args.query, args.gitcookies, - args.limits) def _get_labels_from_args(args): @@ -208,8 +107,12 @@ def main(): # Convert label arguments labels = _get_labels_from_args(args) + # Load authentication credentials + url_opener = create_url_opener_from_args(args) + # Retrieve change lists - change_lists = _get_change_lists_from_args(args) + change_lists = query_change_lists( + url_opener, args.gerrit, args.query, args.limits) if not change_lists: print('error: No matching change lists.', file=sys.stderr) sys.exit(1) @@ -225,8 +128,7 @@ def main(): for change in change_lists: try: res_code, res_json = set_review( - args.gerrit, args.gitcookies, change['id'], labels, - args.message) + url_opener, args.gerrit, change['id'], labels, args.message) except HTTPError as e: res_code = e.code res_json = None