diff --git a/device_deps_regenerator/.gitignore b/device_deps_regenerator/.gitignore new file mode 100644 index 0000000..409a616 --- /dev/null +++ b/device_deps_regenerator/.gitignore @@ -0,0 +1,2 @@ +token +*.json diff --git a/device_deps_regenerator/README.md b/device_deps_regenerator/README.md new file mode 100644 index 0000000..24bc380 --- /dev/null +++ b/device_deps_regenerator/README.md @@ -0,0 +1,6 @@ +1. Use Python 3.2 or higher +2. `pip install -r requirements.txt` +3. grab a new token from [here](https://github.com/settings/tokens) - no scopes needed, just a name. Put it in `token` +4. run `python app.py` to generate the full lineage.dependencies mapping +5. run `python device2kernel.py` to generate kernel -> devices mapping +6. run `python devices.py` to generate device -> dependency mapping (like ../device_deps.json) diff --git a/device_deps_regenerator/app.py b/device_deps_regenerator/app.py new file mode 100644 index 0000000..a39985b --- /dev/null +++ b/device_deps_regenerator/app.py @@ -0,0 +1,126 @@ +import concurrent.futures +import github +import json +import traceback + +from github import Github +from base64 import b64decode + +with open('token') as f: + g = Github(f.readline().strip(), per_page=200) + + +print(g.rate_limiting_resettime) + +org = g.get_organization('LineageOS') + +# supported branches, newest to oldest +CUR_BRANCHES = ['cm-14.1', 'cm-13.0'] + +def get_cm_dependencies(repo): + branch = None + for b in CUR_BRANCHES: + try: + branch = repo.get_branch(b) + break + except github.GithubException: + continue + + if branch is None: + return None + + sha = branch.commit.sha + try: + tree = repo.get_git_tree(sha) + except github.GithubException: + return None + blob_sha = None + for el in tree.tree: + if el.path == 'cm.dependencies' or el.path == 'lineage.dependencies': + blob_sha = el.sha + break + + if blob_sha is None: + return [[], set()] + + blob = repo.get_git_blob(blob_sha) + + deps = b64decode(blob.content) + + cmdeps = json.loads(deps.decode('utf-8')) + + mydeps = [] + non_device_repos = set() + for el in cmdeps: + if '_device_' not in el['repository']: + non_device_repos.add(el['repository']) + depbranch = el.get('branch', branch.name) + mydeps.append({'repo': el['repository'], 'branch': depbranch}) + + return [mydeps, non_device_repos] + +futures = {} +n = 1 + +dependencies = {} +other_repos = set() + + +with concurrent.futures.ThreadPoolExecutor() as executor: + for repo in g.get_organization('LineageOS').get_repos(): + if '_device_' not in repo.name: + continue + print(n, repo.name) + n += 1 + futures[executor.submit(get_cm_dependencies, repo)] = repo.name + for future in concurrent.futures.as_completed(futures): + name = futures[future] + try: + data = future.result() + if data is None: + continue + dependencies[name] = data[0] + other_repos.update(data[1]) + print(name, "=>", data[0]) + except Exception as e: + print('%r generated an exception: %s'%(name, e)) + traceback.print_exc() + continue + futures = {} + + print(other_repos) + for name in other_repos: + print(name) + try: + repo = org.get_repo(name) + futures[executor.submit(get_cm_dependencies, repo)] = name + except Exception: + continue + + other_repos = {} + for future in concurrent.futures.as_completed(futures): + name = futures[future] + try: + data = future.result() + if data is None: + continue + dependencies[name] = data[0] + for el in data[1]: + if el in dependencies: + continue + other_repos.update(data[1]) + print(name, "=>", data[0]) + except Exception as e: + print('%r generated an exception: %s'%(name, e)) + traceback.print_exc() + continue + futures = {} + + +print(other_repos) +#for name in other_repos: +# repo = org.get_repo(name) +# dependencies[name] = get_cm_dependencies(repo) + +with open('out.json', 'w') as f: + json.dump(dependencies, f, indent=4) diff --git a/device_deps_regenerator/device2kernel.py b/device_deps_regenerator/device2kernel.py new file mode 100644 index 0000000..35d2f03 --- /dev/null +++ b/device_deps_regenerator/device2kernel.py @@ -0,0 +1,46 @@ +import json + +# Define device repos that have repos that depend on them, +# otherwise the script will remove these on the assumption +# they are common repos +COMMON_DEVICE = [ + 'android_device_asus_flo', + 'android_device_asus_grouper', + 'android_device_google_marlin', + 'android_device_samsung_espressowifi', + 'android_device_samsung_n1awifi', + 'android_device_samsung_t0lte', +] + +with open('out.json') as f: + mapping = json.load(f) + +kernels = {} + +reverse_deps = {} + +for device in mapping: + deps = mapping[device] + if device not in reverse_deps: + reverse_deps[device] = [] + for repo in deps: + if repo['repo'] not in reverse_deps: + reverse_deps[repo['repo']] = [] + reverse_deps[repo['repo']].append(device) + +def simplify_reverse_deps(repo): + if len(reverse_deps[repo]) == 0 and '-common' not in repo: + return {repo,} + res = set() + for i in reverse_deps[repo]: + res.update(simplify_reverse_deps(i)) + if repo in COMMON_DEVICE: + res.add(repo) + return res + +for repo in reverse_deps: + if 'kernel' in repo: + kernels[repo] = sorted(list(simplify_reverse_deps(repo))) + +with open('kernels.json', 'w') as f: + json.dump(kernels, f, indent=4, sort_keys=True) diff --git a/device_deps_regenerator/devices.py b/device_deps_regenerator/devices.py new file mode 100644 index 0000000..ff44206 --- /dev/null +++ b/device_deps_regenerator/devices.py @@ -0,0 +1,35 @@ +import json + +with open('out.json') as f: + mapping = json.load(f) + +devices = {} +suffixes = {} + +def simplify_reverse_deps(repo, device): + # repo['branch'] = cm-14.1 or cm-14.1-caf or cm-14.1-sony + if 'branch' in repo and repo['branch'].count('-') > 1: # get suffix + if repo['repo'] not in suffixes: + suffixes[repo['repo']] = {} + suffixes[repo['repo']][device] = '-' + repo['branch'].split('-', 2)[2] + + if repo['repo'] not in mapping or len(mapping[repo['repo']]) == 0: + return [repo['repo']] + res = [] + for i in mapping[repo['repo']]: + res += (simplify_reverse_deps(i, device)) + res.append(repo['repo']) + return res + +for repo in mapping: + if 'device' in repo and 'common' not in repo: + codename = repo.split('_', maxsplit=3)[-1] + if codename in devices: + print("warning: dupe: %s"%codename) + + devices[codename] = sorted(list(set(simplify_reverse_deps({'repo': repo}, codename)))) + +with open('devices.json', 'w') as f: + out = {'devices': devices, 'suffixes': suffixes} + out = devices + json.dump(out, f, indent=4, sort_keys=True) diff --git a/device_deps_regenerator/requirements.txt b/device_deps_regenerator/requirements.txt new file mode 100644 index 0000000..b81016f --- /dev/null +++ b/device_deps_regenerator/requirements.txt @@ -0,0 +1 @@ +pygithub