diff --git a/vndk/tools/sourcedr/README.md b/vndk/tools/sourcedr/README.md index da9ed79dd..aad530b36 100644 --- a/vndk/tools/sourcedr/README.md +++ b/vndk/tools/sourcedr/README.md @@ -1,64 +1,8 @@ -# Source Deps Reviewer +# Source Dr. -## Synopsis +This is a collection of source tree analysis tools. -This is a tool for labeling dependencies with a web interface. - -Basically, it greps the specified directory for the given pattern, -and let human reviewers label their dependencies, even code dependencies, -which are code segments that are highly related to the specific pattern. - -## Installation and Dependencies - -This tool depends on [codesearch](https://github.com/google/codesearch) -to generate regular expression index, please install them with: - -``` -$ go get github.com/google/codesearch/cmd/cindex -$ go get github.com/google/codesearch/cmd/csearch -``` - -This tool depends on several Python packages, - -``` -$ pip install -e . -``` - -To run functional test, please do - -``` -$ pip install -e .[dev] -``` - -Prism, a code syntax highlighter is used. -It can be found at https://github.com/PrismJS/prism - -## Usage - -Initialize a project: - -``` -sourcedr init --source-dir [android-src] [project-dir] -``` - -Scan the codebase: - -``` -sourcedr scan -``` - -If there are occurrences that are not reviewed, then review the occurrences -with: - -``` -sourcedr review -``` - -Open browser and visit [http://localhost:5000](http://localhost:5000). - - -## Testing - -``` -$ python3 sourcedr/functional_tests.py -``` +- [blueprint](blueprint) analyzes Android.bp and the dependencies between the + modules. +- [ninja](ninja) analyzes `$(OUT)/combined-${target}.ninja`, which contains all + file dependencies. diff --git a/vndk/tools/sourcedr/requirements.txt b/vndk/tools/sourcedr/requirements.txt deleted file mode 100644 index 9ef2006bc..000000000 --- a/vndk/tools/sourcedr/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -Flask-testing diff --git a/vndk/tools/sourcedr/runtest.py b/vndk/tools/sourcedr/runtest.py deleted file mode 100755 index 7a3e4402e..000000000 --- a/vndk/tools/sourcedr/runtest.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 - -"""Unit tests and functional tests runner.""" - -import argparse -import os -import unittest - - -TESTS_DIR = os.path.join(os.path.dirname(__file__), 'sourcedr', 'tests') - - -def main(): - """ Find and run unit tests and functional tests.""" - - parser = argparse.ArgumentParser() - parser.add_argument('--verbose', '-v', action='store_true') - args = parser.parse_args() - - verbosity = 2 if args.verbose else 1 - - loader = unittest.TestLoader() - tests = loader.discover(TESTS_DIR, 'test_*.py') - runner = unittest.runner.TextTestRunner(verbosity=verbosity) - runner.run(tests) - -if __name__ == '__main__': - main() diff --git a/vndk/tools/sourcedr/setup.py b/vndk/tools/sourcedr/setup.py deleted file mode 100755 index 84ddb50a7..000000000 --- a/vndk/tools/sourcedr/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 - -from setuptools import setup - -setup( - name='sourcedr', - version='1.0', - description='Shared Libs Deps Review Tool', - url='https://android.googlesource.com/platform/development/+' - '/master/vndk/tools/source-deps-reviewer/', - packages=[ - 'blueprint' - 'ninja', - 'sourcedr', - ], - package_data={ - 'sourcedr': [ - 'defaults/pattern_db.csv', - 'defaults/sourcedr.json', - 'static/css/main.css', - 'static/js/main.js', - 'static/prism/css/prism.css', - 'static/prism/js/prism.js', - ], - }, - install_requires=['flask'], - extras_require={ - 'dev': [ - 'flask_testing' - ], - }, - entry_points={ - 'console_scripts': [ - 'sourcedr = sourcedr.commands:main', - ], - } -) diff --git a/vndk/tools/sourcedr/sourcedr/__init__.py b/vndk/tools/sourcedr/sourcedr/__init__.py deleted file mode 100644 index e5a0d9b48..000000000 --- a/vndk/tools/sourcedr/sourcedr/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#!/usr/bin/env python3 diff --git a/vndk/tools/sourcedr/sourcedr/codesearch.py b/vndk/tools/sourcedr/sourcedr/codesearch.py deleted file mode 100644 index b3c970333..000000000 --- a/vndk/tools/sourcedr/sourcedr/codesearch.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env python3 - -"""Code indexing and searching utilities. - -This module will build n-gram file index with codesearch_ and use the index as -a bloom filter to find the regular expression pattern. - -In addition, language-specific filters are provided to ignore matchings in -string literals or comments in the source code. - -.. _codesearch: https://github.com/google/codesearch -""" - -import collections -import os -import re -import subprocess - - -class ClikeFilter(object): - def __init__(self, skip_literals=True, skip_comments=True): - self.file_exts = (b'.c', b'.cpp', b'.cc', b'.cxx', b'.h', b'.hpp', - b'.hxx', b'.java') - self.skip_literals = skip_literals - self.skip_comments = skip_comments - - def process(self, code): - if self.skip_comments: - # Remove // comments. - code = re.sub(b'//[^\\r\\n]*[\\r\\n]', b'', code) - # Remove matched /* */ comments. - code = re.sub(b'/\\*(?:[^*]|(?:\\*+[^*/]))*\\*+/', b'', code) - if self.skip_literals: - # Remove matching quotes. - code = re.sub(b'"(?:\\\\?.)*?"', b'', code) - code = re.sub(b'\'(?:\\\\?.)*?\'', b'', code) - return code - - def get_span(self, code): - span = [] - if self.skip_comments: - # Remove // comments. - p = re.compile(b'//[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - # Remove matched /* */ comments. - p = re.compile(b'/\\*(?:[^*]|(?:\\*+[^*/]))*\\*+/') - for m in p.finditer(code): - span.append(m.span()) - if self.skip_literals: - # Remove matching quotes. - p = re.compile(b'"(?:\\\\?.)*?"') - for m in p.finditer(code): - span.append(m.span()) - p = re.compile(b'\'(?:\\\\?.)*?\'') - for m in p.finditer(code): - span.append(m.span()) - return span - -class PyFilter(object): - def __init__(self, skip_literals=True, skip_comments=True): - self.file_exts = (b'.py',) - self.skip_literals = skip_literals - self.skip_comments = skip_comments - - def process(self, code): - if self.skip_comments: - # Remove # comments - code = re.sub(b'#[^\\r\\n]*[\\r\\n]', b'', code) - if self.skip_literals: - # Remove matching quotes. - code = re.sub(b'"(?:\\\\?.)*?"', b'', code) - code = re.sub(b'\'(?:\\\\?.)*?\'', b'', code) - return code - - def get_span(self, code): - span = [] - if self.skip_comments: - # Remove # comments. - p = re.compile(b'#[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - if self.skip_literals: - # Remove matching quotes. - p = re.compile(b'"(?:\\\\?.)*?"') - for m in p.finditer(code): - span.append(m.span()) - p = re.compile(b'\'(?:\\\\?.)*?\'') - for m in p.finditer(code): - span.append(m.span()) - return span - -class AssemblyFilter(object): - def __init__(self, skip_literals=True, skip_comments=True): - self.file_exts = (b'.s', b'.S') - self.skip_literals = skip_literals - self.skip_comments = skip_comments - - def process(self, code): - if self.skip_comments: - # Remove @ comments - code = re.sub(b'@[^\\r\\n]*[\\r\\n]', b'', code) - # Remove // comments. - code = re.sub(b'//[^\\r\\n]*[\\r\\n]', b'', code) - # Remove matched /* */ comments. - code = re.sub(b'/\\*(?:[^*]|(?:\\*+[^*/]))*\\*+/', b'', code) - return code - - def get_span(self, code): - span = [] - if self.skip_comments: - # Remove # comments. - p = re.compile(b'@[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - # Remove // comments - p = re.compile(b'//[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - # Remove matched /* */ comments - p = re.compile(b'/\\*(?:[^*]|(?:\\*+[^*/]))*\\*+/') - for m in p.finditer(code): - span.append(m.span()) - return span - -class MkFilter(object): - def __init__(self, skip_literals=True, skip_comments=True): - self.file_exts = (b'.mk',) - self.skip_literals = skip_literals - self.skip_comments = skip_comments - - def process(self, code): - if self.skip_comments: - # Remove # comments - code = re.sub(b'#[^\\r\\n]*[\\r\\n]', b'', code) - return code - - def get_span(self, code): - span = [] - if self.skip_comments: - # Remove # comments. - p = re.compile(b'#[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - return span - -class BpFilter(object): - def __init__(self, skip_literals=True, skip_comments=True): - self.file_exts = (b'.bp',) - self.skip_literals = skip_literals - self.skip_comments = skip_comments - - def process(self, code): - if self.skip_comments: - # Remove // comments - code = re.sub(b'//[^\\r\\n]*[\\r\\n]', b'', code) - return code - - def get_span(self, code): - span = [] - if self.skip_comments: - # Remove // comments. - p = re.compile(b'//[^\\r\\n]*[\\r\\n]') - for m in p.finditer(code): - span.append(m.span()) - return span - -class PathFilter(object): - def __init__(self, file_ext_black_list=tuple(), - file_name_black_list=tuple(), - path_component_black_list=tuple()): - self.file_ext_black_list = set( - x.encode('utf-8') for x in file_ext_black_list) - self.file_name_black_list = set( - x.encode('utf-8') for x in file_name_black_list) - self.path_component_black_list = set( - x.encode('utf-8') for x in path_component_black_list) - - def should_skip(self, path): - file_name = os.path.basename(path) - file_ext = os.path.splitext(file_name)[1] - - if file_ext.lower() in self.file_ext_black_list: - return True - if file_name in self.file_name_black_list: - return True - return any(patt in path for patt in self.path_component_black_list) - -class CodeSearch(object): - DEFAULT_NAME = 'csearchindex' - - @classmethod - def get_default_path(cls, project_dir): - return os.path.join(project_dir, 'tmp', cls.DEFAULT_NAME) - - def __init__(self, root_dir, index_file_path, path_filter=None): - self.path = os.path.abspath(index_file_path) - self._root_dir = os.path.abspath(root_dir) - self._env = dict(os.environ) - self._env['CSEARCHINDEX'] = self.path - self._filters = {} - self._path_filter = PathFilter() if path_filter is None else path_filter - - def _run_cindex(self, options): - subprocess.check_call(['cindex'] + options, env=self._env, - cwd=self._root_dir, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - - def _run_csearch(self, options): - if not os.path.exists(self.path): - raise ValueError('Failed to find ' + self.path) - return subprocess.check_output(['csearch'] + options, env=self._env, - cwd=self._root_dir, - stderr=subprocess.DEVNULL) - - def add_filter(self, lang_filter): - for ext in lang_filter.file_exts: - self._filters[ext] = lang_filter - - def add_default_filters(self, skip_literals=True, skip_comments=True): - self.add_filter(ClikeFilter(skip_literals, skip_comments)) - self.add_filter(AssemblyFilter(skip_literals, skip_comments)) - self.add_filter(PyFilter(skip_literals, skip_comments)) - self.add_filter(MkFilter(skip_literals, skip_comments)) - self.add_filter(BpFilter(skip_literals, skip_comments)) - - def build_index(self, remove_existing_index=True): - if remove_existing_index and os.path.exists(self.path): - with contextlib.suppress(FileNotFoundError): - os.remove(self.path) - os.makedirs(os.path.dirname(self.path), exist_ok=True) - self._run_cindex([self._root_dir]) - - def _sanitize_code(self, file_path): - with open(file_path, 'rb') as f: - code = f.read() - file_name = os.path.basename(file_path) - f, ext = os.path.splitext(file_name) - try: - code = self._filters[ext].process(code) - except KeyError: - pass - return code - - def _remove_prefix(self, raw_grep): - ret = b'' - patt = re.compile(b'([^:]+):(\\d+):(.*)$') - for line in raw_grep.split(b'\n'): - match = patt.match(line) - if not match: - continue - file_path = os.path.relpath(match.group(1), - self._root_dir.encode('utf-8')) - line_no = match.group(2) - code = match.group(3) - ret += file_path + b':' + line_no + b':' + code + b'\n' - return ret - - def process_grep(self, raw_grep, pattern, is_regex): - pattern = pattern.encode('utf-8') - if not is_regex: - pattern = re.escape(pattern) - # Limit pattern not to match exceed a line - # Since grep may get multiple patterns in a single entry - pattern = re.compile(pattern + b'[^\\n\\r]*(?:\\n|\\r|$)') - - patt = re.compile(b'([^:]+):(\\d+):(.*)$') - suspect = collections.defaultdict(list) - for line in raw_grep.split(b'\n'): - match = patt.match(line) - if not match: - continue - - file_path = match.group(1) - line_no = match.group(2) - code = match.group(3) - - if self._path_filter.should_skip(file_path): - continue - - abs_file_path = os.path.join(self._root_dir.encode('utf-8'), - file_path) - # Check if any pattern can be found after sanitize_code - if not pattern.search(self._sanitize_code(abs_file_path)): - continue - suspect[abs_file_path].append((file_path, line_no, code)) - - suspect = sorted(suspect.items()) - - processed = b'' - for file_path, entries in suspect: - with open(file_path, 'rb') as f: - code = f.read() - # deep filter - file_name = os.path.basename(file_path) - f, ext = os.path.splitext(file_name) - try: - span = self._filters[ext].get_span(code) - except KeyError: - span = [] - - matchers = [m for m in pattern.finditer(code)] - for i, matcher in enumerate(matchers): - if not span or all(span_ent[0] > matcher.start() or - span_ent[1] <= matcher.start() - for span_ent in span): - processed += (entries[i][0] + b':' + - entries[i][1] + b':' + - entries[i][2] + b'\n') - - return processed - - def raw_grep(self, pattern): - try: - return self._remove_prefix(self._run_csearch(['-n', pattern])) - except subprocess.CalledProcessError as e: - if e.output == b'': - return b'' - raise - - def raw_search(self, pattern, is_regex): - if not is_regex: - pattern = re.escape(pattern) - return self.raw_grep(pattern) diff --git a/vndk/tools/sourcedr/sourcedr/commands/__init__.py b/vndk/tools/sourcedr/sourcedr/commands/__init__.py deleted file mode 100644 index 3a77a6b79..000000000 --- a/vndk/tools/sourcedr/sourcedr/commands/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -"""Parser for command line options.""" - -import argparse -import sys - -from sourcedr.commands import collect, init, scan, review - - -def main(): - """Register sub-commands, parse command line options, and delegate.""" - - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest='subcmd') - commands = {} - - def _register_subcmd(name, init_argparse): - commands[name] = init_argparse(subparsers) - - _register_subcmd('init', init.init_argparse) - _register_subcmd('scan', scan.init_argparse) - _register_subcmd('review', review.init_argparse) - _register_subcmd('collect', collect.init_argparse) - - args = parser.parse_args() - - try: - func = commands[args.subcmd] - except KeyError: - parser.print_help() - sys.exit(1) - - sys.exit(func(args)) - - -if __name__ == '__main__': - main() diff --git a/vndk/tools/sourcedr/sourcedr/commands/collect.py b/vndk/tools/sourcedr/sourcedr/commands/collect.py deleted file mode 100644 index a3b5922ca..000000000 --- a/vndk/tools/sourcedr/sourcedr/commands/collect.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -"""`sourcedr collect` command.""" - -import json -import os - -from sourcedr.project import Project -from sourcedr.map import ( - link_build_dep_and_review_data, load_build_dep_ninja, load_review_data) - - -def init_argparse(parsers): - """Initialize argument parser for `sourcedr collect`.""" - parser = parsers.add_parser('collect', help='Open web-based review UI') - parser.add_argument('input', help='Ninja file') - parser.add_argument('--ninja-deps') - parser.add_argument('--project-dir', default='.') - parser.add_argument('-o', '--output', required=True) - return run - - -def run(args): - project_dir = os.path.expanduser(args.project_dir) - project = Project(project_dir) - - # Load build dependency file - try: - dep = load_build_dep_ninja(args.input, project.source_dir, - args.ninja_deps) - except IOError: - print('error: Failed to open build dependency file:', args.input, - file=sys.stderr) - sys.exit(1) - - # Load review data - table = load_review_data(project.review_db.path) - - # Link build dependency file and review data - res = link_build_dep_and_review_data(dep, table) - - # Write the output file - with open(args.output, 'w') as f: - json.dump(res, f, sort_keys=True, indent=4) - -if __name__ == '__main__': - main() diff --git a/vndk/tools/sourcedr/sourcedr/commands/init.py b/vndk/tools/sourcedr/sourcedr/commands/init.py deleted file mode 100644 index e44812d34..000000000 --- a/vndk/tools/sourcedr/sourcedr/commands/init.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 - -"""`sourcedr init` command.""" - -import os -import sys - -from sourcedr.project import Project - - -def _is_dir_empty(path): - """Determine whether the given path is an empty directory.""" - return len(os.listdir(path)) == 0 - - -def init_argparse(parsers): - """Initialize argument parser for `sourcedr init`.""" - parser = parsers.add_parser('init', help='Start a new review project') - parser.add_argument('--project-dir', default='.') - parser.add_argument('--android-root', required=True, - help='Android source tree root directory') - return run - - -def run(args): - """Main function for `sourcedr init`.""" - - if args.project_dir == '.' and not _is_dir_empty(args.project_dir): - print('error: Current working directory is not an empty directory.', - file=sys.stderr) - - project_dir = os.path.expanduser(args.project_dir) - source_dir = os.path.expanduser(args.android_root) - - Project.get_or_create_project_dir(project_dir, source_dir) diff --git a/vndk/tools/sourcedr/sourcedr/commands/review.py b/vndk/tools/sourcedr/sourcedr/commands/review.py deleted file mode 100644 index 190183a68..000000000 --- a/vndk/tools/sourcedr/sourcedr/commands/review.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -"""`sourcedr review` command.""" - -import os - -from sourcedr.project import Project -from sourcedr.server import create_app - - -def init_argparse(parsers): - """Initialize argument parser for `sourcedr init`.""" - parser = parsers.add_parser('review', help='Open web-based review UI') - parser.add_argument('--project-dir', default='.') - parser.add_argument('--rebuild-csearch-index', action='store_true', - help='Re-build the existing csearch index file') - return run - - -def run(args): - """Main function for `sourcedr init`.""" - project_dir = os.path.expanduser(args.project_dir) - - project = Project(project_dir) - project.update_csearch_index(args.rebuild_csearch_index) - project.update_review_db() - - app = create_app(project) - app.run() diff --git a/vndk/tools/sourcedr/sourcedr/commands/scan.py b/vndk/tools/sourcedr/sourcedr/commands/scan.py deleted file mode 100644 index 6bc5e819b..000000000 --- a/vndk/tools/sourcedr/sourcedr/commands/scan.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -"""`sourcedr scan` command.""" - - -def init_argparse(parsers): - """Initialize argument parser for `sourcedr scan`.""" - parsers.add_parser('scan', help='Scan all pattern occurrences') - return run - - -def run(_): - """Main function for `sourcedr scan`.""" - print('error: Need human review. Run: `sourcedr review`') diff --git a/vndk/tools/sourcedr/sourcedr/defaults/pattern_db.csv b/vndk/tools/sourcedr/sourcedr/defaults/pattern_db.csv deleted file mode 100644 index e5cfc1274..000000000 --- a/vndk/tools/sourcedr/sourcedr/defaults/pattern_db.csv +++ /dev/null @@ -1 +0,0 @@ -1,\bdlopen\b diff --git a/vndk/tools/sourcedr/sourcedr/defaults/sourcedr.json b/vndk/tools/sourcedr/sourcedr/defaults/sourcedr.json deleted file mode 100644 index e2fe014ca..000000000 --- a/vndk/tools/sourcedr/sourcedr/defaults/sourcedr.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "source_dir": "", - "file_ext_blacklist": [ - ".1", - ".ac", - ".cmake", - ".html", - ".info", - ".la", - ".m4", - ".map", - ".md", - ".py", - ".rst", - ".sh", - ".sym", - ".txt", - ".xml" - ], - "file_name_blacklist": [ - "CHANGES.0", - "ChangeLog", - "config.h.in", - "configure", - "configure.in", - "configure.linaro", - "libtool" - ], - "path_component_blacklist": [ - ".git", - ".repo", - "autom4te.cache", - "binutils", - "dejagnu", - "llvm/Config/Config" - ] -} diff --git a/vndk/tools/sourcedr/sourcedr/map.py b/vndk/tools/sourcedr/sourcedr/map.py deleted file mode 100644 index 5659c8668..000000000 --- a/vndk/tools/sourcedr/sourcedr/map.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 - -"""This command maps source file review results to compiled binaries. -""" - -import argparse -import collections -import itertools -import json -import os -import sys - -from ninja import ninja -from sourcedr.review_db import ReviewDB - - -def load_build_dep_graph(graph): - # Collect all shared libraries - shared_libs = set() - for key, value in graph.items(): - if key.split('.')[-1] == 'so': - shared_libs.add(key) - for v in value: - if v.split('.')[-1] == 'so': - shared_libs.add(v) - - # Collect transitive closures - dep = {} - for s in shared_libs: - visited = set() - stack = [s] - while stack: - v = stack.pop() - if v not in visited: - visited.add(v) - try: - stack.extend(x for x in graph[v] - if x not in visited and not x.endswith('.so') - and not x.endswith('.toc')) - except KeyError: - pass - visited.remove(s) - dep[s] = visited - - return dep - - -def load_build_dep_ninja(ninja_path, work_dir, ninja_deps=None): - manifest = ninja.Parser().parse(ninja_path, 'utf-8', ninja_deps) - graph = collections.defaultdict(set) - for build in manifest.builds: - for path in itertools.chain(build.explicit_outs, build.implicit_outs): - ins = graph[path] - ins.update(build.explicit_ins) - ins.update(build.implicit_ins) - ins.update(build.depfile_implicit_ins) - return load_build_dep_graph(graph) - - -def load_build_dep_file(fp): - return load_build_dep_graph(json.load(fp)) - - -def load_build_dep_file_from_path(path): - with open(path, 'r') as fp: - return load_build_dep_file(fp) - - -def load_review_data(path): - table = collections.defaultdict(list) - review_db = ReviewDB(path, None) - for key, item in review_db.data.items(): - table[key.split(':')[0]] += item[0] - return table - - -def link_build_dep_and_review_data(dep, table): - res = collections.defaultdict(list) - for out, ins in dep.items(): - try: - res[out] += table[out] - except KeyError: - pass - - for in_file in ins: - try: - res[out] += table[in_file] - except KeyError: - pass - return res diff --git a/vndk/tools/sourcedr/sourcedr/pattern_db.py b/vndk/tools/sourcedr/sourcedr/pattern_db.py deleted file mode 100644 index b86ae943e..000000000 --- a/vndk/tools/sourcedr/sourcedr/pattern_db.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - - -import os - - -class PatternDB(object): - """Pattern database for patterns to be searched in the source tree. - """ - - DEFAULT_NAME = 'pattern_db.csv' - - - @classmethod - def get_default_path(cls, project_dir): - return os.path.join(project_dir, cls.DEFAULT_NAME) - - - def __init__(self, path): - self.path = path - self.data = self._load() - - - def _load(self): - with open(self.path, 'r') as f: - patterns = [] - is_regexs = [] - for line in f: - line = line.rstrip('\n') - sp = line.split(',') - is_regexs.append(sp[0]) - patterns.append(','.join(sp[1:])) - return (patterns, is_regexs) - - - def load(self): - self.data = self._load() - return self.data - - - def save_new_pattern(self, patt, is_regex): - """Add a pattern to the database.""" - with open(self.path, 'a') as f: - f.write(str(int(is_regex)) + ',' + patt + '\n') diff --git a/vndk/tools/sourcedr/sourcedr/project.py b/vndk/tools/sourcedr/sourcedr/project.py deleted file mode 100644 index f6115bda8..000000000 --- a/vndk/tools/sourcedr/sourcedr/project.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 - -"""SourceDR project configurations and databases. - -`Project` class holds configuration files, review databases, pattern databases, -and `codesearch` index files. -""" - -import collections -import json -import os -import shutil - -from sourcedr.codesearch import CodeSearch, PathFilter -from sourcedr.pattern_db import PatternDB -from sourcedr.review_db import ReviewDB -from sourcedr.utils import LockedFile - - -class Config(object): - """SourceDR project configuration file.""" - - DEFAULT_NAME = 'sourcedr.json' - - _PATH_TRAVERSAL_ATTRS = ( - 'file_ext_blacklist', 'file_name_blacklist', - 'path_component_blacklist') - - - @classmethod - def get_default_path(cls, project_dir): - """Get the default path of the configuration file under a project - directory.""" - return os.path.join(project_dir, cls.DEFAULT_NAME) - - - def __init__(self, path): - self.path = path - - self.source_dir = None - self.file_ext_blacklist = set() - self.file_name_blacklist = set() - self.path_component_blacklist = set() - - - def load(self): - """Load the project configuration from the JSON file.""" - with open(self.path, 'r') as config_fp: - config_json = json.load(config_fp) - for key, value in config_json.items(): - if key == 'source_dir': - self.source_dir = value - elif key in self._PATH_TRAVERSAL_ATTRS: - setattr(self, key, set(value)) - else: - raise ValueError('unknown config name: ' + key) - - - def save(self): - """Save the project configuration to the JSON file.""" - with LockedFile(self.path, 'x') as config_fp: - config = collections.OrderedDict() - config['source_dir'] = self.source_dir - for key in self._PATH_TRAVERSAL_ATTRS: - config[key] = sorted(getattr(self, key)) - json.dump(config, config_fp, indent=2) - - -class Project(object): - """SourceDR project configuration files and databases.""" - - def __init__(self, project_dir): - """Load a project from a given project directory.""" - - project_dir = os.path.abspath(project_dir) - self.project_dir = project_dir - - if not os.path.isdir(project_dir): - raise ValueError('project directory not found: ' + project_dir) - - # Load configuration files - config_path = Config.get_default_path(project_dir) - self.config = Config(config_path) - self.config.load() - - # Recalculate source directory - self.source_dir = os.path.abspath( - os.path.join(project_dir, self.config.source_dir)) - - # csearchindex file - path_filter = PathFilter(self.config.file_ext_blacklist, - self.config.file_name_blacklist, - self.config.path_component_blacklist) - csearch_index_path = CodeSearch.get_default_path(project_dir) - self.codesearch = CodeSearch(self.source_dir, csearch_index_path, - path_filter) - self.codesearch.add_default_filters() - - # Review database file - review_db_path = ReviewDB.get_default_path(project_dir) - self.review_db = ReviewDB(review_db_path, self.codesearch) - - # Pattern database file - pattern_db_path = PatternDB.get_default_path(project_dir) - self.pattern_db = PatternDB(pattern_db_path) - - # Sanity checks - self._check_source_dir() - self._check_lock_files() - - - def update_csearch_index(self, remove_existing_index): - """Create or update codesearch index.""" - self.codesearch.build_index(remove_existing_index) - - - def update_review_db(self): - """Update the entries in the review database.""" - patterns, is_regexs = self.pattern_db.load() - self.review_db.find(patterns, is_regexs) - - - def _check_source_dir(self): - """Check the availability of the source directory.""" - if not os.path.isdir(self.source_dir): - raise ValueError('source directory not found: ' + self.source_dir) - - - def _check_lock_files(self): - """Check whether there are some lock files.""" - for path in (self.config.path, self.review_db.path, - self.pattern_db.path): - if LockedFile.is_locked(path): - raise ValueError('file locked: ' + path) - - - @classmethod - def create_project_dir(cls, project_dir, source_dir): - """Create a directory for a new project and setup default - configurations.""" - - if not os.path.isdir(source_dir): - raise ValueError('source directory not found: ' + source_dir) - - os.makedirs(project_dir, exist_ok=True) - - # Compute the relative path between project_dir and source_dir - project_dir = os.path.abspath(project_dir) - source_dir = os.path.relpath(os.path.abspath(source_dir), project_dir) - - # Copy default files - defaults_dir = os.path.join(os.path.dirname(__file__), 'defaults') - for name in (Config.DEFAULT_NAME, PatternDB.DEFAULT_NAME): - shutil.copyfile(os.path.join(defaults_dir, name), - os.path.join(project_dir, name)) - - # Update the source directory in the configuration file - config_path = Config.get_default_path(project_dir) - config = Config(config_path) - config.load() - config.source_dir = source_dir - config.save() - - return Project(project_dir) - - - @classmethod - def get_or_create_project_dir(cls, project_dir, source_dir): - config_file_path = Config.get_default_path(project_dir) - if os.path.exists(config_file_path): - return Project(project_dir) - else: - return cls.create_project_dir(project_dir, source_dir) diff --git a/vndk/tools/sourcedr/sourcedr/review_db.py b/vndk/tools/sourcedr/sourcedr/review_db.py deleted file mode 100644 index d72965f78..000000000 --- a/vndk/tools/sourcedr/sourcedr/review_db.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import re - - -class ReviewDB(object): - DEFAULT_NAME = 'review_db.json' - - - @classmethod - def get_default_path(cls, project_dir): - return os.path.join(project_dir, cls.DEFAULT_NAME) - - - def __init__(self, path, codesearch): - self.path = path - self._cs = codesearch - try: - self.data = self._load_data() - except FileNotFoundError: - self.data = {} - - - # patterns and is_regexs are lists - def find(self, patterns, is_regexs): - # they shouldn't be empty - assert patterns and is_regexs - processed = b'' - for pattern, is_regex in zip(patterns, is_regexs): - if not is_regex: - pattern = re.escape(pattern) - raw_grep = self._cs.raw_grep(pattern) - if raw_grep == b'': - continue - processed += self._cs.process_grep(raw_grep, pattern, is_regex) - self.to_json(processed) - - - def add_pattern(self, pattern, is_regex): - if not is_regex: - pattern = re.escape(pattern) - raw_grep = self._cs.raw_grep(pattern) - if raw_grep == b'': - return - processed = self._cs.process_grep(raw_grep, pattern, is_regex) - self.add_to_json(processed) - - - def to_json(self, processed): - data = {} - patt = re.compile('([^:]+):(\\d+):(.*)$') - for line in processed.decode('utf-8').split('\n'): - match = patt.match(line) - if not match: - continue - data[line] = ([], []) - - # if old data exists, perform merge - if os.path.exists(self.path): - data.update(self._load_data()) - - self._save_data(data) - self.data = self._load_data() - - - def add_to_json(self, processed): - # Load all matched grep. - data = self._load_data() - patt = re.compile('([^:]+):(\\d+):(.*)$') - for line in processed.decode('utf-8').split('\n'): - match = patt.match(line) - if not match: - continue - data[line] = ([], []) - - self._save_data(data) - self.data = self._load_data() - - - def add_label(self, label, deps, codes): - self.data[label] = (deps, codes) - self._save_data(self.data) - - - def _save_data(self, data): - with open(self.path, 'w') as data_fp: - json.dump(data, data_fp, sort_keys=True, indent=4) - - - def _load_data(self): - with open(self.path, 'r') as data_fp: - return json.load(data_fp) diff --git a/vndk/tools/sourcedr/sourcedr/server.py b/vndk/tools/sourcedr/sourcedr/server.py deleted file mode 100644 index 5469efe14..000000000 --- a/vndk/tools/sourcedr/sourcedr/server.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 - -import collections -import functools -import json -import os -import re - -from flask import ( - Blueprint, Flask, current_app, jsonify, render_template, request) - - -codereview = Blueprint('codereview', '__name__', 'templates') - - -# whether the code segment is exactly in file -def same(fl, code, source_dir): - fl = os.path.join(source_dir, fl) - with open(fl, 'r') as f: - fc = f.read() - return code in fc - - -# check if the file needes to be reiewed again -def check(codes, source_dir): - ret = [] - for item in codes: - fl = item.split(':')[0] - code = item[len(fl) + 1:] - ret.append(same(fl, code, source_dir)) - return ret - - -@codereview.route('/get_started') -def _get_started(): - project = current_app.config.project - source_dir = project.source_dir - review_db = project.review_db - - lst, done= [], [] - for key, item in sorted(review_db.data.items()): - lst.append(key) - if item[0]: - done.append(all(check(item[1], source_dir))) - else: - done.append(False) - - pattern_lst = project.pattern_db.load()[0] - abs_path = os.path.abspath(source_dir) - - return jsonify(lst=json.dumps(lst), - done=json.dumps(done), - pattern_lst=json.dumps(pattern_lst), - path_prefix=os.path.join(abs_path, '')) - - -@codereview.route('/load_file') -def _load_file(): - project = current_app.config.project - source_dir = project.source_dir - review_db = project.review_db - - path = request.args.get('path') - - if path not in review_db.data.keys(): - print('No such entry', path) - return jsonify(result='') - deps, codes = review_db.data[path] - - return jsonify(deps=json.dumps(deps), codes=json.dumps(codes), - okays=json.dumps(check(codes, source_dir))) - - -@codereview.route('/get_file') -def _get_file(): - path = request.args.get('path') - path = os.path.join(current_app.config.project.source_dir, path) - - if not os.path.exists(path): - return jsonify(result='No such file') - with open(path, 'r') as f: - code = f.read() - - return jsonify(result=code) - - -@codereview.route('/save_all') -def _save_all(): - label = request.args.get('label') - deps = json.loads(request.args.get('deps')) - codes = json.loads(request.args.get('codes')) - - project = current_app.config.project - review_db = project.review_db - review_db.add_label(label, deps, codes) - - return jsonify(result='done') - - -# This function add pattern to grep -@codereview.route('/add_pattern') -def _add_pattern(): - patt = request.args.get('pattern') - is_regex = request.args.get('is_regex') - engine = current_app.config.project.review_db - engine.add_pattern(patt, is_regex) - - project = current_app.config.project - project.pattern_db.save_new_pattern(patt, is_regex) - return jsonify(result='done') - - -# This function does a temporary grep to the directory -# Not adding the result to database -@codereview.route('/temporary_search') -def _temporary_search(): - path = request.args.get('path') - patt = request.args.get('pattern') - is_regex = request.args.get('is_regex') - codesearch = current_app.config.project.codesearch - result = codesearch.raw_search(patt, is_regex).decode('utf-8') - dic = collections.defaultdict(list) - patt = re.compile('([^:]+):(\\d+):(.*)$') - for line in result.split('\n'): - match = patt.match(line) - if not match: - continue - - file_path = match.group(1) - line_no = match.group(2) - code = match.group(3) - dic[file_path].append((line_no, code)) - - def compare(item1, item2): - key1, value1 = item1 - key2, value2 = item2 - cnt1 = os.path.commonprefix([path, key1]).count('/') - cnt2 = os.path.commonprefix([path, key2]).count('/') - e1 = os.path.relpath(key1, path).count('/') - e2 = os.path.relpath(key2, path).count('/') - # prefer smaller edit distance - if e1 < e2: return -1 - if e2 < e1: return 1 - # prefer deeper common ancestor - if cnt1 > cnt2: return -1 - if cnt2 > cnt1: return 1 - # lexicographical order - if key1 < key2: return -1 - if key2 < key1: return 1 - return 0 - - result = sorted(dic.items(), key=functools.cmp_to_key(compare)) - return jsonify(result=json.dumps(result)) - - -@codereview.route('/') -def render(): - return render_template('index.html') - - -def create_app(project): - app = Flask(__name__) - app.register_blueprint(codereview) - app.config.project = project - return app diff --git a/vndk/tools/sourcedr/sourcedr/static/css/main.css b/vndk/tools/sourcedr/sourcedr/static/css/main.css deleted file mode 100644 index 11bb7bcbc..000000000 --- a/vndk/tools/sourcedr/sourcedr/static/css/main.css +++ /dev/null @@ -1,26 +0,0 @@ -h3, h4 { - display: inline-block; -} - -pre { - white-space: pre-wrap; /* Since CSS 2.1 */ - white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ - white-space: -pre-wrap; /* Opera 4-6 */ - white-space: -o-pre-wrap; /* Opera 7 */ - word-wrap: break-word; /* Internet Explorer 5.5+ */ - background-color: #ffffff; - margin: 0; -} - -@media (min-width: 768px) { - .modal-xl { - width: 90%; - max-width:1200px; - } -} - -.affix { - top:50px; - right:0; - position:fixed; -} diff --git a/vndk/tools/sourcedr/sourcedr/static/js/main.js b/vndk/tools/sourcedr/sourcedr/static/js/main.js deleted file mode 100644 index e8057052b..000000000 --- a/vndk/tools/sourcedr/sourcedr/static/js/main.js +++ /dev/null @@ -1,333 +0,0 @@ -(function () { - 'use strict'; - - var ccounter = 0; - var counter = 0; - var current_item = null; - - // make item list sortable - $( function() { - $("#item_list").sortable(); - $("#item_list").disableSelection(); - }); - - function moveToTop(index) { - if (index == 0) { - return; - } - let target = $('#item_list').children().eq(index); - let tp = $('#item_list').children().eq(0); - let old_offset = target.position(); - tp.before(target); - let new_offset = target.position(); - let tmp = target.clone().appendTo('#item_list') - .css('position', 'absolute') - .css('left', old_offset.left) - .css('top', old_offset.top); - target.hide(); - let new_pos = {'top': new_offset.top, 'left': new_offset.left}; - tmp.animate(new_pos, 'slow', function() { - target.show(); - tmp.remove(); - }); - } - - function getSelText() { - let txt = window.getSelection(); - $('#selected_text').val(txt); - $('#code_file_path').val($('#browsing_file_path').text()); - return txt; - } - - function taskHtml(text, cnt) { - return '
blocks are highlighted multiple times. It is necessary
- * to cleanup any left-over tags, because the whitespace inside of the
- * tags change the content of the tag.
- */
- var num = 0;
- $$('.line-highlight', pre).forEach(function (line) {
- num += line.textContent.length;
- line.parentNode.removeChild(line);
- });
-
- // Remove extra whitespace
- if (num && /^( \n)+$/.test(env.code.slice(-num))) {
- env.code = env.code.slice(0, -num);
- }
-});
-
-Prism.hooks.add('complete', function (env) {
- if (!env.code) {
- return;
- }
-
- // works only for wrapped inside (not inline)
- var pre = env.element.parentNode;
- var clsReg = /\s*\bline-numbers\b\s*/;
- if (
- !pre || !/pre/i.test(pre.nodeName) ||
- // Abort only if nor the nor the have the class
- (!clsReg.test(pre.className) && !clsReg.test(env.element.className))
- ) {
- return;
- }
-
- if (env.element.querySelector(".line-numbers-rows")) {
- // Abort if line numbers already exists
- return;
- }
-
- if (clsReg.test(env.element.className)) {
- // Remove the class "line-numbers" from the
- env.element.className = env.element.className.replace(clsReg, '');
- }
- if (!clsReg.test(pre.className)) {
- // Add the class "line-numbers" to the
- pre.className += ' line-numbers';
- }
-
- var match = env.code.match(/\n(?!$)/g);
- var linesNum = match ? match.length + 1 : 1;
- var lineNumbersWrapper;
-
- var lines = '';
- for (let i = 1; i < linesNum + 1; i++) {
- lines += '';
- }
-
- lineNumbersWrapper = document.createElement('span');
- lineNumbersWrapper.setAttribute('aria-hidden', 'true');
- lineNumbersWrapper.className = 'line-numbers-rows';
- lineNumbersWrapper.innerHTML = lines;
-
- if (pre.hasAttribute('data-start')) {
- pre.style.counterReset = 'linenumber ' + (parseInt(pre.getAttribute('data-start'), 10) - 1);
- }
-
- env.element.appendChild(lineNumbersWrapper);
-
-});
-
-Prism.hooks.add('complete', function(env) {
- var pre = env.element.parentNode;
- var lines = pre && pre.getAttribute('data-line');
-
- if (!pre || !lines || !/pre/i.test(pre.nodeName)) {
- return;
- }
-
- clearTimeout(fakeTimer);
-
- highlightLines(pre, lines);
-
- fakeTimer = setTimeout(applyHash, 1);
-});
-
-if(window.addEventListener) {
- window.addEventListener('hashchange', applyHash);
-}
-
-})();
-
-(function() {
-
-if (typeof self === 'undefined' || !self.Prism || !self.document) {
- return;
-}
-
-}());
diff --git a/vndk/tools/sourcedr/sourcedr/templates/index.html b/vndk/tools/sourcedr/sourcedr/templates/index.html
deleted file mode 100644
index f03ced3ac..000000000
--- a/vndk/tools/sourcedr/sourcedr/templates/index.html
+++ /dev/null
@@ -1,111 +0,0 @@
-
-
-
- Source Deps Reviewer
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Code review tool
-
-
-
- Browsing:
-
-
-
-
-
-
-
-
- Temporary search
-
- Add patterns to grep
-
-
-
-
- File labeling:
-
- Pattern line number:
-
- Library Dependencies
-
-
- Code Dependencies
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/vndk/tools/sourcedr/sourcedr/tests/__init__.py b/vndk/tools/sourcedr/sourcedr/tests/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/vndk/tools/sourcedr/sourcedr/tests/test_map.py b/vndk/tools/sourcedr/sourcedr/tests/test_map.py
deleted file mode 100644
index c30a89842..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/test_map.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import unittest
-
-from sourcedr.map import (
- link_build_dep_and_review_data, load_build_dep_file_from_path,
- load_build_dep_ninja, load_review_data)
-
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')
-
-
-class MapTest(unittest.TestCase):
- MAP_TESTDATA_DIR = os.path.join(TESTDATA_DIR, 'map')
- DEP_PATH = os.path.join(MAP_TESTDATA_DIR, 'build_dep.json')
- REVIEW_DB_PATH = os.path.join(MAP_TESTDATA_DIR, 'data.json')
- NINJA_PATH = os.path.join(MAP_TESTDATA_DIR, 'build.ninja')
- NINJA_DEP_PATH = os.path.join(MAP_TESTDATA_DIR, 'ninja_deps')
-
-
- def test_load_build_dep_file(self):
- dep = load_build_dep_file_from_path(self.DEP_PATH)
-
- self.assertIn('liba.so', dep)
- self.assertIn('libb.so', dep)
- self.assertIn('libc.so', dep)
-
- self.assertSetEqual({'a.h', 'a1.c', 'a1.o', 'a2.c', 'a2.o'},
- dep['liba.so'])
- self.assertSetEqual({'a.h', 'b.c', 'b.o'}, dep['libb.so'])
- self.assertSetEqual({'c.c', 'c.o'}, dep['libc.so'])
-
-
- def test_load_build_dep_ninja(self):
- dep = load_build_dep_ninja(self.NINJA_PATH, self.MAP_TESTDATA_DIR,
- self.NINJA_DEP_PATH)
-
- self.assertIn('liba.so', dep)
- self.assertIn('libb.so', dep)
- self.assertIn('libc.so', dep)
-
- self.assertSetEqual({'a.h', 'a1.c', 'a1.o', 'a2.c', 'a2.o'},
- dep['liba.so'])
- self.assertSetEqual({'a.h', 'b.c', 'b.o'}, dep['libb.so'])
- self.assertSetEqual({'c.c', 'c.o'}, dep['libc.so'])
-
-
- def test_load_review_data(self):
- data = load_review_data(self.REVIEW_DB_PATH)
- self.assertIn('a.h', data)
- self.assertEqual(['libx.so'], data['a.h'])
-
-
- def test_link_build_dep_and_review_data(self):
- dep = load_build_dep_file_from_path(self.DEP_PATH)
- data = load_review_data(self.REVIEW_DB_PATH)
- result = link_build_dep_and_review_data(dep, data)
-
- self.assertIn('liba.so', result)
- self.assertIn('libb.so', result)
- self.assertIn('libc.so', result)
-
- self.assertEqual(['libx.so'], result['liba.so'])
- self.assertEqual(['libx.so'], result['libb.so'])
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/vndk/tools/sourcedr/sourcedr/tests/test_project.py b/vndk/tools/sourcedr/sourcedr/tests/test_project.py
deleted file mode 100644
index 10afc5ae9..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/test_project.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import tempfile
-import unittest
-
-from sourcedr.project import Config
-
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')
-
-
-class ConfigTest(unittest.TestCase):
- PROJECT_DIR = os.path.join(TESTDATA_DIR, 'project')
- CONFIG_PATH = os.path.join(PROJECT_DIR, Config.DEFAULT_NAME)
-
-
- def test_load(self):
- config = Config(self.CONFIG_PATH)
- config.load()
- self.assertEqual('path/to/android/src', config.source_dir)
-
-
- def test_save(self):
- with tempfile.TemporaryDirectory(prefix='test_sourcedr_') as tmp_dir:
- config_path = Config.get_default_path(tmp_dir)
- config = Config(config_path)
- config.source_dir = 'path/to/android/src'
- config.save()
- with open(config_path, 'r') as actual_fp:
- actual = actual_fp.read().strip()
- with open(self.CONFIG_PATH, 'r') as expected_fp:
- expected = expected_fp.read().strip()
- self.assertEqual(actual, expected)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/vndk/tools/sourcedr/sourcedr/tests/test_review_db.py b/vndk/tools/sourcedr/sourcedr/tests/test_review_db.py
deleted file mode 100644
index d1cdc096d..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/test_review_db.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import unittest
-
-from sourcedr.codesearch import CodeSearch
-from sourcedr.review_db import ReviewDB
-
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')
-ANDROID_DIR = os.path.join(TESTDATA_DIR, 'android_src')
-
-
-class ReviewDBTest(unittest.TestCase):
- def setUp(self):
- self.csearch_index_path = 'csearchindex'
- self.review_db_path = ReviewDB.DEFAULT_NAME
-
-
- def tearDown(self):
- os.remove(self.csearch_index_path)
- os.remove(self.review_db_path)
-
-
- def test_preprocess(self):
- codesearch = CodeSearch(ANDROID_DIR, self.csearch_index_path)
- codesearch.build_index()
- review_db = ReviewDB(ReviewDB.DEFAULT_NAME, codesearch)
- review_db.find(patterns=['dlopen'], is_regexs=[False])
- self.assertTrue(os.path.exists(ReviewDB.DEFAULT_NAME))
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/vndk/tools/sourcedr/sourcedr/tests/test_server.py b/vndk/tools/sourcedr/sourcedr/tests/test_server.py
deleted file mode 100644
index 9a8a46fce..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/test_server.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import tempfile
-import unittest
-
-import flask_testing
-
-from sourcedr.project import Project
-from sourcedr.review_db import ReviewDB
-from sourcedr.server import create_app
-
-
-TESTDATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')
-ANDROID_DIR = os.path.join(TESTDATA_DIR, 'android_src')
-
-
-class ViewTest(flask_testing.TestCase):
- def create_app(self):
- self.tmp_dir = tempfile.TemporaryDirectory(prefix='test_sourcedr_')
- project = Project.get_or_create_project_dir(
- self.tmp_dir.name, ANDROID_DIR)
- project.update_csearch_index(True)
- self.project = project
-
- app = create_app(project)
- app.config['TESTING'] = True
- self.app = app
- return app
-
-
- def setUp(self):
- review_db = self.project.review_db
- review_db.find(patterns=['dlopen'], is_regexs=[False])
-
-
- def tearDown(self):
- self.tmp_dir.cleanup()
-
-
- def test_get_file(self):
- test_arg = 'example.c'
- response = self.client.get('/get_file',
- query_string=dict(path=test_arg))
- ret = response.json['result']
- with open(os.path.join(ANDROID_DIR, test_arg), 'r') as f:
- self.assertEqual(ret, f.read())
-
-
- def test_load_file(self):
- test_arg = 'dlopen/test.c'
- test_arg += ':10: handle = dlopen("libm.so.6", RTLD_LAZY);'
- response = self.client.get('/load_file',
- query_string=dict(path=test_arg))
- deps = json.loads(response.json['deps'])
- codes = json.loads(response.json['codes'])
- with open(self.project.review_db.path, 'r') as f:
- cdata = json.load(f)
-
- self.assertEqual(deps, cdata[test_arg][0])
- self.assertEqual(codes, cdata[test_arg][1])
-
-
- def test_save_all(self):
- label = os.path.abspath('sourcedr/test/dlopen/test.c')
- label += ':10: handle = dlopen("libm.so.6", RTLD_LAZY);'
- test_arg = {
- 'label': label,
- 'deps': json.dumps(['this_is_a_test.so']),
- 'codes': json.dumps(['arr_0', 'arr_1'])
- }
- response = self.client.get('/save_all', query_string=test_arg)
- cdata = ReviewDB(self.project.review_db.path, None).data
- self.assertEqual(['this_is_a_test.so'], cdata[test_arg['label']][0])
- self.assertEqual(['arr_0', 'arr_1'], cdata[test_arg['label']][1])
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/build.ninja b/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/build.ninja
deleted file mode 100644
index 2cc71de35..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/build.ninja
+++ /dev/null
@@ -1,15 +0,0 @@
-rule cc
- command = gcc -c -o $out $in -MMD -MF $out.d
- deps = gcc
- depfile = $out.d
-
-rule ld
- command = gcc -o $out $in
-
-build example.o: cc example.c
-
-build example.so: ld example.o
-
-build dlopen/test.o: cc dlopen/test.c
-
-build dlopen/test.so: ld dlopen/test.o
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/dlopen/test.c b/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/dlopen/test.c
deleted file mode 100644
index c3055698c..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/dlopen/test.c
+++ /dev/null
@@ -1,24 +0,0 @@
-#include
-#include
-#include
-
-int main(int argc, char **argv) {
- void *handle;
- double (*cosine)(double);
- char *error;
-
- handle = dlopen("libm.so.6", RTLD_LAZY);
- if (!handle) {
- fputs (dlerror(), stderr);
- exit(1);
- }
-
- cosine = dlsym(handle, "cos");
- if ((error = dlerror()) != NULL) {
- fputs(error, stderr);
- exit(1);
- }
-
- printf ("%f\n", (*cosine)(2.0));
- dlclose(handle);
-}
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.c b/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.c
deleted file mode 100644
index f8e329f91..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.c
+++ /dev/null
@@ -1,10 +0,0 @@
-int main() {
- printf("This is a simple testing file\n");
- int dlopen_analysis = 1;
- "This line with dlopen shouldn't be found"
- /*
- * This dlopen shouldn't be found
- */
- dlopen("dlopen");
- handle = dlopen("libm.so.6", RTLD_LAZY);
-}
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.txt b/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.txt
deleted file mode 100644
index 27f787eca..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/android_src/example.txt
+++ /dev/null
@@ -1 +0,0 @@
-dlopen() in .txt file should not be matched
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build.ninja b/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build.ninja
deleted file mode 100644
index 5f732ac36..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build.ninja
+++ /dev/null
@@ -1,21 +0,0 @@
-rule cc
- command = gcc -c -o $out $in -MMD -MF $out.d
- deps = gcc
- depfile = $out.d
-
-rule ld
- command = gcc -o $out $in
-
-build liba.so: ld a1.o a2.o
-
-build libb.so: ld b.o
-
-build libc.so: ld c.o
-
-build a1.o: cc a1.c
-
-build a2.o: cc a2.c
-
-build b.o: cc b.c
-
-build c.o: cc c.c
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build_dep.json b/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build_dep.json
deleted file mode 100644
index d8b6f49e2..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/build_dep.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "liba.so": ["libb.so", "libc.so", "a1.o", "a2.o"],
- "libb.so": ["b.o"],
- "libc.so": ["c.o"],
- "a1.o": ["a.h", "a1.c"],
- "a2.o": ["a.h", "a2.c"],
- "b.o": ["a.h", "b.c"],
- "c.o": ["c.c"]
-}
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/data.json b/vndk/tools/sourcedr/sourcedr/tests/testdata/map/data.json
deleted file mode 100644
index 0d1bfe287..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/data.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "a.h:2:dlopen(\"libx.so\",": [
- ["libx.so"],
- ["a.h:2:dlopen(\"libx.so\","]
- ]
-}
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/ninja_deps b/vndk/tools/sourcedr/sourcedr/tests/testdata/map/ninja_deps
deleted file mode 100644
index 78414b331..000000000
Binary files a/vndk/tools/sourcedr/sourcedr/tests/testdata/map/ninja_deps and /dev/null differ
diff --git a/vndk/tools/sourcedr/sourcedr/tests/testdata/project/sourcedr.json b/vndk/tools/sourcedr/sourcedr/tests/testdata/project/sourcedr.json
deleted file mode 100644
index e9126d899..000000000
--- a/vndk/tools/sourcedr/sourcedr/tests/testdata/project/sourcedr.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "source_dir": "path/to/android/src",
- "file_ext_blacklist": [],
- "file_name_blacklist": [],
- "path_component_blacklist": []
-}
diff --git a/vndk/tools/sourcedr/sourcedr/utils.py b/vndk/tools/sourcedr/sourcedr/utils.py
deleted file mode 100644
index 168e437fe..000000000
--- a/vndk/tools/sourcedr/sourcedr/utils.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-
-"""Utility functions or classes."""
-
-import os
-
-
-class LockedFile(object): # pylint: disable=too-few-public-methods
- """Open a file with `.lock` file and rename it if everything goes well."""
-
-
- def __init__(self, path, mode):
- assert 'x' in mode
- self._path = path
- self._mode = mode
- self._fp = None
-
-
- def __enter__(self):
- """Open the file at the specified path and with specified mode."""
- self._fp = open(self._get_locked_path(self._path), self._mode)
- return self._fp
-
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- """Close the file object and rename the file if there are no
- exceptions."""
- self._fp.close()
- self._fp = None
- if exc_val is None:
- os.rename(self._get_locked_path(self._path), self._path)
-
-
- @classmethod
- def _get_locked_path(cls, path):
- """Get the file path for the `.lock` file."""
- return path + '.lock'
-
-
- @classmethod
- def is_locked(cls, path):
- """Check whether a path is locked."""
- return os.path.exists(cls._get_locked_path(path))