LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge changes from topic "check-vndk-dep2"

Browse Source 
* changes:
  sourcedr: check_vndk_dep should evaluate recursively
  sourcedr: check_vndk_dep should scan dirs when necessary
  sourcedr: Refactor the logic in glob matcher
  sourcedr: Add integer support to check_vndk_dep
  sourcedr: Skip the modules without names
  sourcedr: Fix checks on hidl_interface
  sourcedr: Treat proprietary modules as vendor modules
This commit is contained in:
Logan Chien
2018-02-27 02:55:09 +00:00
committed by Gerrit Code Review
parent b41803118b 887ff6dbf0
commit 8dd2bafcff
4 changed files with 489 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
vndk/tools/source-deps-reviewer/sourcedr/blueprint/blueprint.py
View File

@@ -108,9 +108,10 @@ class Token(Enum): # pylint: disable=too-few-public-methods
PLUS = 11
COMMA = 12
STRING = 13
INTEGER = 14
COMMENT = 14
SPACE = 15
COMMENT = 15
SPACE = 16
class LexerError(ValueError):
@@ -330,6 +331,7 @@ class Lexer(object):
(Token.PLUS, '\\+'),
(Token.COMMA, ','),
(Token.STRING, '["`]'),
(Token.INTEGER, '-{0,1}[0-9]+'),
(Token.COMMENT,
'/(?:(?:/[^\\n]*)|(?:\\*(?:(?:[^*]*)|(?:\\*+[^/*]))*\\*+/))'),
@@ -364,7 +366,10 @@ class Lexer(object):
end, literal = cls.lex_string(buf, offset)
else:
end = match.end()
literal = buf[offset:end] if token == Token.IDENT else None
if token in {Token.IDENT, Token.INTEGER}:
literal = buf[offset:end]
else:
literal = None
return (token, end, literal)
@@ -427,6 +432,49 @@ class Bool(Expr): # pylint: disable=too-few-public-methods
return self
class Integer(Expr): # pylint: disable=too-few-public-methods
"""Integer constant literal."""
__slots__ = ('value',)
def __init__(self, value):
"""Create an integer constant literal."""
self.value = value
def __repr__(self):
"""Convert an integer constant literal to string representation."""
return repr(self.value)
def __bool__(self):
"""Convert an integer constant literal to Python bool type."""
return bool(self.value)
__nonzero__ = __bool__
def __int__(self):
"""Convert an integer constant literal to Python int type."""
return self.value
def __eq__(self, rhs):
"""Compare whether two instances are equal."""
return self.value == rhs.value
def __hash__(self):
"""Compute the hashed value."""
return hash(self.value)
def eval(self, env):
"""Evaluate the integer expression under an environment."""
return self
class VarRef(Expr): # pylint: disable=too-few-public-methods
"""A reference to a variable."""
@@ -472,13 +520,13 @@ class Dict(Expr, collections.OrderedDict):
class Concat(Expr): # pylint: disable=too-few-public-methods
"""List/string concatenation operator."""
"""List/string/integer plus operator."""
__slots__ = ('lhs', 'rhs')
def __init__(self, lhs, rhs):
"""Create a list concatenation expression."""
"""Create a list/string/integer plus expression."""
self.lhs = lhs
self.rhs = rhs
@@ -488,14 +536,16 @@ class Concat(Expr): # pylint: disable=too-few-public-methods
def eval(self, env):
"""Evaluate list concatenation operator under an environment."""
"""Evaluate list/string/integer plus operator under an environment."""
lhs = self.lhs.eval(env)
rhs = self.rhs.eval(env)
if isinstance(lhs, List) and isinstance(rhs, List):
return List(itertools.chain(lhs, rhs))
if isinstance(lhs, String) and isinstance(rhs, String):
return String(lhs + rhs)
raise TypeError('bad concatenation')
if isinstance(lhs, Integer) and isinstance(rhs, Integer):
return Integer(int(lhs) + int(rhs))
raise TypeError('bad plus operands')
#------------------------------------------------------------------------------
@@ -608,6 +658,14 @@ class Parser(object):
return string
def parse_integer(self):
"""Parse an integer."""
lexer = self.lexer
integer = Integer(int(lexer.literal))
lexer.consume(Token.INTEGER)
return integer
def parse_operand(self):
"""Parse an operand."""
lexer = self.lexer
@@ -616,6 +674,8 @@ class Parser(object):
return self.parse_string()
if token == Token.IDENT:
return self.parse_ident_rvalue()
if token == Token.INTEGER:
return self.parse_integer()
if token == Token.LBRACKET:
return self.parse_list()
if token == Token.LBRACE:
@@ -701,17 +761,17 @@ class RecursiveParser(object):
wildcards."""
for path in glob.glob(pattern):
if os.path.isfile(path) and os.path.basename(path) == sub_file_name:
yield path
continue
sub_file_path = os.path.join(path, sub_file_name)
if os.path.isfile(sub_file_path):
yield sub_file_path
if os.path.isfile(path):
if os.path.basename(path) == sub_file_name:
yield path
else:
sub_file_path = os.path.join(path, sub_file_name)
if os.path.isfile(sub_file_path):
yield sub_file_path
@classmethod
def find_sub_files_from_env(cls, rootdir, env,
def find_sub_files_from_env(cls, rootdir, env, use_subdirs,
default_sub_name='Android.bp'):
"""Find the sub files from the names specified in build, subdirs, and
optional_subdirs."""
@@ -721,24 +781,23 @@ class RecursiveParser(object):
if 'build' in env:
subs.extend(os.path.join(rootdir, filename)
for filename in env['build'].eval(env))
if use_subdirs:
sub_name = env['subname'] if 'subname' in env else default_sub_name
sub_name = env['subname'] if 'subname' in env else default_sub_name
if 'subdirs' in env:
for path in env['subdirs'].eval(env):
subs.extend(
cls.glob_sub_files(os.path.join(rootdir, path), sub_name))
if 'optional_subdirs' in env:
for path in env['optional_subdirs'].eval(env):
subs.extend(
cls.glob_sub_files(os.path.join(rootdir, path), sub_name))
if 'subdirs' in env:
for path in env['subdirs'].eval(env):
subs.extend(cls.glob_sub_files(os.path.join(rootdir, path),
sub_name))
if 'optional_subdirs' in env:
for path in env['optional_subdirs'].eval(env):
subs.extend(cls.glob_sub_files(os.path.join(rootdir, path),
sub_name))
return subs
@staticmethod
def _parse_one_file(path, env):
def _read_file(path, env):
"""Read a blueprint file and return modules and the environment."""
with open(path, 'r') as bp_file:
content = bp_file.read()
parser = Parser(Lexer(content), env)
@@ -747,18 +806,25 @@ class RecursiveParser(object):
def _parse_file(self, path, env, evaluate):
"""Parse blueprint files recursively."""
self.visited.add(os.path.abspath(path))
modules, sub_env = self._parse_one_file(path, env)
"""Parse a blueprint file and append to self.modules."""
modules, sub_env = self._read_file(path, env)
if evaluate:
modules = [(ident, attrs.eval(env)) for ident, attrs in modules]
self.modules += modules
return sub_env
def _parse_file_recursive(self, path, env, evaluate, use_subdirs):
"""Parse a blueprint file and recursively."""
self.visited.add(os.path.abspath(path))
sub_env = self._parse_file(path, env, evaluate)
rootdir = os.path.dirname(path)
sub_file_paths = self.find_sub_files_from_env(rootdir, sub_env)
sub_file_paths = self.find_sub_files_from_env(rootdir, sub_env,
use_subdirs)
sub_env.pop('build', None)
sub_env.pop('subdirs', None)
@@ -766,18 +832,106 @@ class RecursiveParser(object):
for sub_file_path in sub_file_paths:
if os.path.abspath(sub_file_path) not in self.visited:
self._parse_file(sub_file_path, sub_env, evaluate)
self._parse_file_recursive(sub_file_path, sub_env, evaluate,
use_subdirs)
return sub_env
def _scan_and_parse_all_file_recursive(self, filename, path, env, evaluate):
"""Scan all files with the specified name and parse them."""
rootdir = os.path.dirname(path)
envs = [(rootdir, env)]
assert env is not None
# Scan directories for all blueprint files
for basedir, dirnames, filenames in os.walk(rootdir):
# Drop irrelevant environments
while not basedir.startswith(envs[-1][0]):
envs.pop()
# Filter sub directories
new_dirnames = []
for name in dirnames:
if name in {'.git', '.repo'}:
continue
if basedir == rootdir and name == 'out':
continue
new_dirnames.append(name)
dirnames[:] = new_dirnames
# Parse blueprint files
if filename in filenames:
try:
path = os.path.join(basedir, filename)
sys.stdout.flush()
sub_env = self._parse_file_recursive(path, envs[-1][1],
evaluate, False)
assert sub_env is not None
envs.append((basedir, sub_env))
except IOError:
pass
def parse_file(self, path, env=None, evaluate=True):
"""Parse blueprint files recursively."""
self._parse_file(path, {} if env is None else env, evaluate)
if env is None:
env = {}
sub_env = self._read_file(path, env)[1]
if 'subdirs' in sub_env or 'optional_subdirs' in sub_env:
self._parse_file_recursive(path, env, evaluate, True)
else:
self._scan_and_parse_all_file_recursive('Android.bp', path, env,
evaluate)
#------------------------------------------------------------------------------
# Transformation
#------------------------------------------------------------------------------
def _build_named_modules_dict(modules):
"""Build a name-to-module dict."""
named_modules = {}
for i, (ident, attrs) in enumerate(modules):
name = attrs.get('name')
if name is not None:
named_modules[name] = [ident, attrs, i]
return named_modules
def _po_sorted_modules(modules, named_modules):
"""Sort modules in post order."""
modules = [(ident, attrs, i) for i, (ident, attrs) in enumerate(modules)]
# Build module dependency graph.
edges = {}
for ident, attrs, module_id in modules:
defaults = attrs.get('defaults')
if defaults:
edges[module_id] = set(
named_modules[default][2] for default in defaults)
# Traverse module graph in post order.
post_order = []
visited = set()
def _traverse(module_id):
visited.add(module_id)
for next_module_id in edges.get(module_id, []):
if next_module_id not in visited:
_traverse(next_module_id)
post_order.append(modules[module_id])
for module_id in range(len(modules)):
if module_id not in visited:
_traverse(module_id)
return post_order
def evaluate_default(attrs, default_attrs):
"""Add default attributes if the keys do not exist."""
for key, value in default_attrs.items():
@@ -792,14 +946,9 @@ def evaluate_default(attrs, default_attrs):
def evaluate_defaults(modules):
"""Add default attributes to all modules if the keys do not exist."""
mods = {}
for ident, attrs in modules:
mods[attrs['name']] = (ident, attrs)
for i, (ident, attrs) in enumerate(modules):
defaults = attrs.get('defaults')
if defaults is None:
continue
for default in defaults:
attrs = evaluate_default(attrs, mods[default][1])
named_modules = _build_named_modules_dict(modules)
for ident, attrs, i in _po_sorted_modules(modules, named_modules):
for default in attrs.get('defaults', []):
attrs = evaluate_default(attrs, named_modules[default][1])
modules[i] = (ident, attrs)
return modules

15
vndk/tools/source-deps-reviewer/sourcedr/blueprint/check_vndk_dep.py
View File

@@ -47,7 +47,7 @@ def _is_vndk_sp(module):
def _is_vendor(module):
"""Get the `vendor` module property."""
try:
return bool(module['vendor'])
return module.get('vendor', False) or module.get('proprietary', False)
except KeyError:
return False
@@ -87,17 +87,28 @@ def _build_module_dict(modules):
"""Build module dictionaries that map module names to modules."""
all_libs = {}
llndk_libs = {}
for rule, module in modules:
name = module['name']
name = module.get('name')
if name is None:
continue
if rule == 'llndk_library':
llndk_libs[name] = (rule, module)
if rule in {'llndk_library', 'ndk_library'}:
continue
if rule.endswith('_library') or \
rule.endswith('_library_shared') or \
rule.endswith('_library_static') or \
rule.endswith('_headers'):
all_libs[name] = (rule, module)
if rule == 'hidl_interface':
all_libs[name] = (rule, module)
all_libs[name + '-adapter-helper'] = (rule, module)
module['vendor_available'] = True
return (all_libs, llndk_libs)

61
vndk/tools/source-deps-reviewer/sourcedr/blueprint/tests/test_ast.py
View File

@@ -20,7 +20,7 @@
import unittest
from blueprint import Bool, Concat, Dict, Expr, List, String, VarRef
from blueprint import Bool, Concat, Dict, Expr, Integer, List, String, VarRef
#------------------------------------------------------------------------------
@@ -91,6 +91,65 @@ class BoolTest(unittest.TestCase):
self.assertEqual('True', repr(Bool(True)))
#------------------------------------------------------------------------------
# Integer
#------------------------------------------------------------------------------
class IntegerTest(unittest.TestCase):
"""Unit tests for the Integer class."""
def test_int(self):
"""Test Integer.__init__(), Integer.__bool__(), Integer.__int__(), and
Integer.eval() methods."""
expr = Integer(0)
self.assertFalse(bool(expr))
self.assertEqual(0, int(expr))
self.assertEqual(0, int(expr.eval({})))
expr = Integer(1)
self.assertTrue(bool(expr))
self.assertEqual(1, int(expr))
self.assertEqual(1, int(expr.eval({})))
expr = Integer(2)
self.assertTrue(bool(expr))
self.assertEqual(2, int(expr))
self.assertEqual(2, int(expr.eval({})))
def test_equal(self):
"""Test Integer.__eq__() method."""
expr_zero1 = Integer(0)
expr_zero2 = Integer(0)
expr_one1 = Integer(1)
expr_one2 = Integer(1)
self.assertIsNot(expr_zero1, expr_zero2)
self.assertEqual(expr_zero1, expr_zero2)
self.assertIsNot(expr_one1, expr_one2)
self.assertEqual(expr_one1, expr_one2)
def test_hash(self):
"""Test Integer.__hash__() method."""
expr_zero = Integer(0)
expr_one = Integer(1)
self.assertEqual(hash(Integer(False)), hash(expr_zero))
self.assertEqual(hash(Integer(True)), hash(expr_one))
def test_repr(self):
"""Test Integer.__repr__() method."""
self.assertEqual('0', repr(Integer(0)))
self.assertEqual('1', repr(Integer(1)))
#------------------------------------------------------------------------------
# String
#------------------------------------------------------------------------------

222
vndk/tools/source-deps-reviewer/sourcedr/blueprint/tests/test_evaluate_default.py Normal file
View File

@@ -0,0 +1,222 @@
#!/usr/bin/env python3
#
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""This module contains the unit tests to check evaluate_default(s)."""
import unittest
from blueprint import Dict, evaluate_default, evaluate_defaults
#------------------------------------------------------------------------------
# Evaluate Default
#------------------------------------------------------------------------------
class EvaluateDefaultTest(unittest.TestCase):
"""Test cases for evaluate_default()."""
def test_evaluate_default(self):
"""Test evaluate_default()."""
attrs = {'a': 'specified_a', 'b': 'specified_b'}
default_attrs = {'a': 'default_a', 'c': 'default_c'}
result = evaluate_default(attrs, default_attrs)
self.assertEqual(len(result), 3)
self.assertEqual(result['a'], 'specified_a')
self.assertEqual(result['b'], 'specified_b')
self.assertEqual(result['c'], 'default_c')
def test_evaluate_default_nested(self):
"""Test evaluate_default() with nested properties."""
attrs = {'c': Dict({'a': 'specified_a'})}
default_attrs = {'c': Dict({'a': 'default_a', 'b': 'default_b'})}
result = evaluate_default(attrs, default_attrs)
self.assertEqual(len(result), 1)
self.assertEqual(len(result['c']), 2)
self.assertEqual(result['c']['a'], 'specified_a')
self.assertEqual(result['c']['b'], 'default_b')
#------------------------------------------------------------------------------
# Evaluate Defaults
#------------------------------------------------------------------------------
class EvaluateDefaultsTest(unittest.TestCase):
"""Test cases for evaluate_defaults()."""
def test_evaluate_defaults(self):
"""Test evaluate_defaults()."""
modules = [
('cc_defaults', {
'name': 'libfoo-defaults',
'a': 'default_a',
'b': 'default_b',
}),
('cc_library', {
'name': 'libfoo',
'defaults': ['libfoo-defaults'],
'a': 'specified_a',
}),
]
modules = evaluate_defaults(modules)
module = modules[-1][1]
self.assertEqual(module['name'], 'libfoo')
self.assertEqual(module['a'], 'specified_a')
self.assertEqual(module['b'], 'default_b')
def test_evaluate_two_defaults(self):
"""Test evaluate_defaults() with two defaults."""
modules = [
('cc_defaults', {
'name': 'libfoo-defaults',
'a': 'libfoo_default_a',
'b': 'libfoo_default_b',
}),
('cc_defaults', {
'name': 'libbar-defaults',
'a': 'libbar_default_a',
'b': 'libbar_default_b',
'c': 'libbar_default_c',
}),
('cc_library', {
'name': 'libfoo',
'defaults': ['libfoo-defaults', 'libbar-defaults'],
'a': 'specified_a',
}),
]
modules = evaluate_defaults(modules)
module = modules[-1][1]
self.assertEqual(module['name'], 'libfoo')
self.assertEqual(module['a'], 'specified_a')
self.assertEqual(module['b'], 'libfoo_default_b')
self.assertEqual(module['c'], 'libbar_default_c')
def test_skip_modules_without_name(self):
"""Test whether evaluate_defaults() skips modules without names."""
modules = [('special_rules', {})]
try:
modules = evaluate_defaults(modules)
except KeyError:
self.fail('modules without names must not cause KeyErrors')
def test_evaluate_recursive(self):
"""Test whether evaluate_defaults() can evaluate defaults
recursively."""
modules = [
('cc_defaults', {
'name': 'libfoo-defaults',
'defaults': ['libtest-defaults'],
'a': 'libfoo_default_a',
'b': 'libfoo_default_b',
}),
('cc_defaults', {
'name': 'libbar-defaults',
'a': 'libbar_default_a',
'b': 'libbar_default_b',
'c': 'libbar_default_c',
'd': 'libbar_default_d',
}),
('cc_defaults', {
'name': 'libtest-defaults',
'a': 'libtest_default_a',
'b': 'libtest_default_b',
'c': 'libtest_default_c',
'e': 'libtest_default_e',
}),
('cc_library', {
'name': 'libfoo',
'defaults': ['libfoo-defaults', 'libbar-defaults'],
'a': 'specified_a',
}),
]
modules = evaluate_defaults(modules)
module = modules[-1][1]
self.assertEqual(module['name'], 'libfoo')
self.assertEqual(module['a'], 'specified_a')
self.assertEqual(module['b'], 'libfoo_default_b')
self.assertEqual(module['c'], 'libtest_default_c')
self.assertEqual(module['d'], 'libbar_default_d')
self.assertEqual(module['e'], 'libtest_default_e')
def test_evaluate_recursive_diamond(self):
"""Test whether evaluate_defaults() can evaluate diamond defaults
recursively."""
modules = [
('cc_defaults', {
'name': 'libfoo-defaults',
'defaults': ['libtest-defaults'],
'a': 'libfoo_default_a',
'b': 'libfoo_default_b',
}),
('cc_defaults', {
'name': 'libbar-defaults',
'defaults': ['libtest-defaults'],
'a': 'libbar_default_a',
'b': 'libbar_default_b',
'c': 'libbar_default_c',
'd': 'libbar_default_d',
}),
('cc_defaults', {
'name': 'libtest-defaults',
'a': 'libtest_default_a',
'b': 'libtest_default_b',
'c': 'libtest_default_c',
'e': 'libtest_default_e',
}),
('cc_library', {
'name': 'libfoo',
'defaults': ['libfoo-defaults', 'libbar-defaults'],
'a': 'specified_a',
}),
]
modules = evaluate_defaults(modules)
module = modules[-1][1]
self.assertEqual(module['name'], 'libfoo')
self.assertEqual(module['a'], 'specified_a')
self.assertEqual(module['b'], 'libfoo_default_b')
self.assertEqual(module['c'], 'libtest_default_c')
self.assertEqual(module['d'], 'libbar_default_d')
self.assertEqual(module['e'], 'libtest_default_e')
if __name__ == '__main__':
unittest.main()