Move device_deps scripts to LineageOS/scripts [2/2]

Change-Id: Idb6d986f1dd18a1cd272d5b6c5c6b14ae6fb6b2b

device_deps_regenerator/.gitignore

@@ -0,0 +1,2 @@
+token
+*.json

device_deps_regenerator/README.md

@@ -0,0 +1,6 @@
+1. Use Python 3.2 or higher
+2. `pip install -r requirements.txt`
+3. grab a new token from [here](https://github.com/settings/tokens) - no scopes needed, just a name. Put it in `token`
+4. run `python app.py` to generate the full lineage.dependencies mapping
+5. run `python device2kernel.py` to generate kernel -> devices mapping
+6. run `python devices.py` to generate device -> dependency mapping (like ../device_deps.json)

device_deps_regenerator/app.py

@@ -0,0 +1,126 @@
+import concurrent.futures
+import github
+import json
+import traceback
+
+from github import Github
+from base64 import b64decode
+
+with open('token') as f:
+    g = Github(f.readline().strip(), per_page=200)
+
+
+print(g.rate_limiting_resettime)
+
+org = g.get_organization('LineageOS')
+
+# supported branches, newest to oldest
+CUR_BRANCHES = ['cm-14.1', 'cm-13.0']
+
+def get_cm_dependencies(repo):
+    branch = None
+    for b in CUR_BRANCHES:
+        try:
+            branch = repo.get_branch(b)
+            break
+        except github.GithubException:
+            continue
+
+    if branch is None:
+        return None
+
+    sha = branch.commit.sha
+    try:
+        tree = repo.get_git_tree(sha)
+    except github.GithubException:
+        return None
+    blob_sha = None
+    for el in tree.tree:
+        if el.path == 'cm.dependencies' or el.path == 'lineage.dependencies':
+            blob_sha = el.sha
+            break
+
+    if blob_sha is None:
+        return [[], set()]
+
+    blob = repo.get_git_blob(blob_sha)
+
+    deps = b64decode(blob.content)
+
+    cmdeps = json.loads(deps.decode('utf-8'))
+
+    mydeps = []
+    non_device_repos = set()
+    for el in cmdeps:
+        if '_device_' not in el['repository']:
+            non_device_repos.add(el['repository'])
+        depbranch = el.get('branch', branch.name)
+        mydeps.append({'repo': el['repository'], 'branch': depbranch})
+
+    return [mydeps, non_device_repos]
+
+futures = {}
+n = 1
+
+dependencies = {}
+other_repos = set()
+
+
+with concurrent.futures.ThreadPoolExecutor() as executor:
+    for repo in g.get_organization('LineageOS').get_repos():
+        if '_device_' not in repo.name:
+            continue
+        print(n, repo.name)
+        n += 1
+        futures[executor.submit(get_cm_dependencies, repo)] = repo.name
+    for future in concurrent.futures.as_completed(futures):
+        name = futures[future]
+        try:
+            data = future.result()
+            if data is None:
+                continue
+            dependencies[name] = data[0]
+            other_repos.update(data[1])
+            print(name, "=>", data[0])
+        except Exception as e:
+            print('%r generated an exception: %s'%(name, e))
+            traceback.print_exc()
+            continue
+    futures = {}
+
+    print(other_repos)
+    for name in other_repos:
+        print(name)
+        try:
+            repo = org.get_repo(name)
+            futures[executor.submit(get_cm_dependencies, repo)] = name
+        except Exception:
+            continue
+
+    other_repos = {}
+    for future in concurrent.futures.as_completed(futures):
+        name = futures[future]
+        try:
+            data = future.result()
+            if data is None:
+                continue
+            dependencies[name] = data[0]
+            for el in data[1]:
+                if el in dependencies:
+                    continue
+                other_repos.update(data[1])
+            print(name, "=>", data[0])
+        except Exception as e:
+            print('%r generated an exception: %s'%(name, e))
+            traceback.print_exc()
+            continue
+    futures = {}
+
+
+print(other_repos)
+#for name in other_repos:
+#    repo = org.get_repo(name)
+#    dependencies[name] = get_cm_dependencies(repo)
+
+with open('out.json', 'w') as f:
+    json.dump(dependencies, f, indent=4)

device_deps_regenerator/device2kernel.py

@@ -0,0 +1,46 @@
+import json
+
+# Define device repos that have repos that depend on them,
+# otherwise the script will remove these on the assumption
+# they are common repos
+COMMON_DEVICE = [
+    'android_device_asus_flo',
+    'android_device_asus_grouper',
+    'android_device_google_marlin',
+    'android_device_samsung_espressowifi',
+    'android_device_samsung_n1awifi',
+    'android_device_samsung_t0lte',
+]
+
+with open('out.json') as f:
+    mapping = json.load(f)
+
+kernels = {}
+
+reverse_deps = {}
+
+for device in mapping:
+    deps = mapping[device]
+    if device not in reverse_deps:
+        reverse_deps[device] = []
+    for repo in deps:
+        if repo['repo'] not in reverse_deps:
+            reverse_deps[repo['repo']] = []
+        reverse_deps[repo['repo']].append(device)
+
+def simplify_reverse_deps(repo):
+    if len(reverse_deps[repo]) == 0 and '-common' not in repo:
+        return {repo,}
+    res = set()
+    for i in reverse_deps[repo]:
+        res.update(simplify_reverse_deps(i))
+    if repo in COMMON_DEVICE:
+        res.add(repo)
+    return res
+
+for repo in reverse_deps:
+    if 'kernel' in repo:
+        kernels[repo] = sorted(list(simplify_reverse_deps(repo)))
+
+with open('kernels.json', 'w') as f:
+    json.dump(kernels, f, indent=4, sort_keys=True)

device_deps_regenerator/devices.py

@@ -0,0 +1,35 @@
+import json
+
+with open('out.json') as f:
+    mapping = json.load(f)
+
+devices = {}
+suffixes = {}
+
+def simplify_reverse_deps(repo, device):
+    # repo['branch'] = cm-14.1 or cm-14.1-caf or cm-14.1-sony
+    if 'branch' in repo and repo['branch'].count('-') > 1: # get suffix
+        if repo['repo'] not in suffixes:
+            suffixes[repo['repo']] = {}
+        suffixes[repo['repo']][device] = '-' + repo['branch'].split('-', 2)[2]
+
+    if repo['repo'] not in mapping or len(mapping[repo['repo']]) == 0:
+        return [repo['repo']]
+    res = []
+    for i in mapping[repo['repo']]:
+        res += (simplify_reverse_deps(i, device))
+    res.append(repo['repo'])
+    return res
+
+for repo in mapping:
+    if 'device' in repo and 'common' not in repo:
+        codename = repo.split('_', maxsplit=3)[-1]
+        if codename in devices:
+            print("warning: dupe: %s"%codename)
+
+        devices[codename] = sorted(list(set(simplify_reverse_deps({'repo': repo}, codename))))
+
+with open('devices.json', 'w') as f:
+    out = {'devices': devices, 'suffixes': suffixes}
+    out = devices
+    json.dump(out, f, indent=4, sort_keys=True)

device_deps_regenerator/requirements.txt

@@ -0,0 +1 @@
+pygithub