Merge changes Ia1b282bd,I3c53a5d1

* changes:
  vndk-def: Code cleanup
  vndk-def: Remove unused command
This commit is contained in:
Logan Chien
2019-01-02 04:19:13 +00:00
committed by Gerrit Code Review
2 changed files with 260 additions and 314 deletions

View File

@@ -0,0 +1,8 @@
[MESSAGES CONTROL]
disable=
fixme,
invalid-name,
missing-docstring,
ungrouped-imports,
useless-object-inheritance,

View File

@@ -39,7 +39,7 @@ else:
def makedirs(path, exist_ok):
if exist_ok and os.path.isdir(path):
return
return os.makedirs(path)
os.makedirs(path)
class mmap(mmap):
def __enter__(self):
@@ -50,14 +50,14 @@ else:
def __getitem__(self, key):
res = super(mmap, self).__getitem__(key)
if type(key) == int:
if isinstance(key, int):
return ord(res)
return res
class Py3Bytes(bytes):
def __getitem__(self, key):
res = super(Py3Bytes, self).__getitem__(key)
if type(key) == int:
if isinstance(key, int):
return ord(res)
return Py3Bytes(res)
@@ -135,7 +135,8 @@ def decode_mutf8(input, errors='strict'):
raise UnicodeDecodeError('mutf-8', input, start, i + 1, reason)
def raise_surrogate_error(start, reason):
raise UnicodeSurrogateDecodeError('mutf-8', input, start, i + 1, reason)
raise UnicodeSurrogateDecodeError(
'mutf-8', input, start, i + 1, reason)
for i, byte in enumerate_bytes(input):
if (byte & 0x80) == 0x00:
@@ -165,7 +166,7 @@ def decode_mutf8(input, errors='strict'):
raise_error(i, 'invalid start byte')
if num_next == 0:
if code >= 0xd800 and code <= 0xdbff: # High surrogate
if 0xd800 <= code <= 0xdbff: # High surrogate
if code_surrogate is not None:
raise_surrogate_error(
start_surrogate, 'invalid high surrogate')
@@ -173,10 +174,11 @@ def decode_mutf8(input, errors='strict'):
start_surrogate = start
continue
if code >= 0xdc00 and code <= 0xdfff: # Low surrogate
if 0xdc00 <= code <= 0xdfff: # Low surrogate
if code_surrogate is None:
raise_surrogate_error(start, 'invalid low surrogate')
code = ((code_surrogate & 0x3f) << 10) | (code & 0x3f) + 0x10000
code = (((code_surrogate & 0x3f) << 10) |
(code & 0x3f) + 0x10000)
code_surrogate = None
start_surrogate = None
elif code_surrogate is not None:
@@ -314,7 +316,7 @@ class Elf_Sym(collections.namedtuple(
@property
def st_bind(self):
return (self.st_info >> 4)
return self.st_info >> 4
@property
def is_local(self):
@@ -583,7 +585,8 @@ class ELF(object):
if self.is_32bit:
def parse_elf_sym(offset):
return parse_struct(Elf_Sym, elf_sym_fmt, offset, 'bad elf sym')
return parse_struct(
Elf_Sym, elf_sym_fmt, offset, 'bad elf sym')
else:
def parse_elf_sym(offset):
try:
@@ -659,7 +662,8 @@ class ELF(object):
# Parse entries in .dynamic section.
assert struct.calcsize(elf_dyn_fmt) == dynamic_shdr.sh_entsize
dynamic_end = dynamic_off + dynamic_shdr.sh_size
for ent_off in range(dynamic_off, dynamic_end, dynamic_shdr.sh_entsize):
for ent_off in range(
dynamic_off, dynamic_end, dynamic_shdr.sh_entsize):
ent = parse_elf_dyn(ent_off)
if ent.d_tag == ELF.DT_NEEDED:
self.dt_needed.append(extract_str(dynstr_off + ent.d_val))
@@ -712,7 +716,7 @@ class ELF(object):
for line_no, line in enumerate(lines):
match = patt.match(line)
if not match:
print('error: {}: {}: failed to parse'
print('error: {}: {}: Failed to parse'
.format(path, line_no + 1), file=sys.stderr)
continue
key = match.group(1)
@@ -903,7 +907,8 @@ class DexFileReader(object):
for name in cls.generate_classes_dex_names():
try:
with zip_file.open(name) as dex_file:
for s in cls.enumerate_dex_strings_buf(dex_file.read()):
dex_file_content = dex_file.read()
for s in cls.enumerate_dex_strings_buf(dex_file_content):
yield s
except KeyError:
break
@@ -977,9 +982,9 @@ class DexFileReader(object):
# Parse vdex file header (w.r.t. version)
if version == b'000\x00':
VdexHeader = cls.VdexHeader0
elif version >= b'001\x00' and version < b'016\x00':
elif b'001\x00' <= version < b'016\x00':
VdexHeader = cls.VdexHeader1
elif version >= b'016\x00' and version < b'019\x00':
elif b'016\x00' <= version < b'019\x00':
VdexHeader = cls.VdexHeader16
elif version == b'019\x00':
VdexHeader = cls.VdexHeader19
@@ -1012,13 +1017,13 @@ class DexFileReader(object):
else:
quickening_table_off_size = 0
for i in range(vdex_header.number_of_dex_files):
for _ in range(vdex_header.number_of_dex_files):
# Skip quickening_table_off size
offset += quickening_table_off_size
# Check the dex file magic
dex_magic = buf[offset:offset + 4]
if dex_magic != b'dex\n' and dex_magic != b'cdex':
if dex_magic not in (b'dex\n', b'cdex'):
raise ValueError('bad dex file offset {}'.format(offset))
dex_header = cls.Header.unpack_from(buf, offset)
@@ -1358,7 +1363,7 @@ def is_accessible(path):
def scan_accessible_files(root):
for base, dirs, files in os.walk(root):
for base, _, files in os.walk(root):
for filename in files:
path = os.path.join(base, filename)
if is_accessible(path):
@@ -1415,8 +1420,7 @@ NUM_PARTITIONS = 2
SPLibResult = collections.namedtuple(
'SPLibResult',
'sp_hal sp_hal_dep vndk_sp_hal ll_ndk ll_ndk_indirect '
'vndk_sp_both')
'sp_hal sp_hal_dep vndk_sp_hal ll_ndk ll_ndk_indirect vndk_sp_both')
VNDKLibTuple = defaultnamedtuple('VNDKLibTuple', 'vndk_sp vndk', [])
@@ -1450,11 +1454,10 @@ class VNDKLibDir(list):
"""Extract VNDK version from a name."""
if name in {'vndk', 'vndk-sp'}:
return 'current'
elif name.startswith('vndk-sp-'):
if name.startswith('vndk-sp-'):
return name[len('vndk-sp-'):]
elif name.startswith('vndk-'):
if name.startswith('vndk-'):
return name[len('vndk-'):]
else:
return None
@@ -1462,7 +1465,7 @@ class VNDKLibDir(list):
def extract_path_component(cls, path, index):
"""Extract n-th path component from a posix path."""
start = 0
for i in range(index):
for _ in range(index):
pos = path.find('/', start)
if pos == -1:
return None
@@ -1606,7 +1609,8 @@ class VNDKLibDir(list):
path = os.path.join(vendor_dir, 'default.prop')
try:
with open(path, 'r') as property_file:
result = cls._get_property(property_file, 'ro.vndk.version')
result = cls._get_property(
property_file, 'ro.vndk.version')
if result is not None:
return result
except FileNotFoundError:
@@ -1988,7 +1992,8 @@ class ELFLinker(object):
if alter_subdirs:
alter_patt = ELFLinker._compile_path_matcher(root, alter_subdirs)
if ignored_subdirs:
ignored_patt = ELFLinker._compile_path_matcher(root, ignored_subdirs)
ignored_patt = ELFLinker._compile_path_matcher(
root, ignored_subdirs)
for path, elf in scan_elf_files(root, unzip_files):
# Ignore ELF files with unknown machine ID (eg. DSP).
@@ -1996,10 +2001,11 @@ class ELFLinker(object):
continue
# Ignore ELF files with matched path.
short_path = os.path.join('/', partition_name, path[prefix_len:])
if ignored_subdirs and ignored_patt.match(path):
continue
short_path = os.path.join('/', partition_name, path[prefix_len:])
if alter_subdirs and alter_patt.match(path):
self.add_lib(alter_partition, short_path, elf)
else:
@@ -2047,7 +2053,8 @@ class ELFLinker(object):
candidates = list(resolver.get_candidates(
lib.path, dt_needed, lib.elf.dt_rpath, lib.elf.dt_runpath))
print('warning: {}: Missing needed library: {} Tried: {}'
.format(lib.path, dt_needed, candidates), file=sys.stderr)
.format(lib.path, dt_needed, candidates),
file=sys.stderr)
lib.unresolved_dt_needed.append(dt_needed)
continue
lib.add_needed_dep(dep)
@@ -2134,9 +2141,10 @@ class ELFLinker(object):
for version in vndk_lib_dirs:
vndk_sp_dirs, vndk_dirs = \
vndk_lib_dirs.create_vndk_search_paths(lib_dir, version)
vndk_sp_libs = system_vndk_sp_libs[version] | \
vendor_vndk_sp_libs[version]
search_paths = self._get_vndk_sp_search_paths(lib_dir, vndk_sp_dirs)
vndk_sp_libs = \
system_vndk_sp_libs[version] | vendor_vndk_sp_libs[version]
search_paths = self._get_vndk_sp_search_paths(
lib_dir, vndk_sp_dirs)
resolver = ELFResolver(lib_dict, search_paths)
self._resolve_lib_set_deps(vndk_sp_libs, resolver, generic_refs)
@@ -2257,8 +2265,7 @@ class ELFLinker(object):
predefined_vndk_sp_indirect = set(
lib for lib in self.all_libs() if lib.is_vndk_sp_indirect)
predefined_vndk_sp_indirect_private = set(
lib for lib in self.all_libs()
if lib.is_vndk_sp_indirect_private)
lib for lib in self.all_libs() if lib.is_vndk_sp_indirect_private)
# FIXME: Don't squash VNDK-SP-Indirect-Private into VNDK-SP-Indirect.
predefined_vndk_sp_indirect |= predefined_vndk_sp_indirect_private
@@ -2329,7 +2336,8 @@ class ELFLinker(object):
vndk_sp_unused, is_not_vndk_sp_indirect_unused, True)
vndk_sp_unused_deps -= vndk_sp_unused
vndk_sp_indirect_unused = set(lib for lib in predefined_vndk_sp_indirect
vndk_sp_indirect_unused = set(
lib for lib in predefined_vndk_sp_indirect
if VNDKLibDir.is_in_vndk_sp_dir(lib.path))
vndk_sp_indirect_unused -= vndk_sp_indirect
vndk_sp_indirect_unused -= vndk_sp_unused
@@ -2338,8 +2346,8 @@ class ELFLinker(object):
# TODO: Compute VNDK-SP-Indirect-Private.
vndk_sp_indirect_private = set()
assert not (vndk_sp & vndk_sp_indirect)
assert not (vndk_sp_unused & vndk_sp_indirect_unused)
assert not vndk_sp & vndk_sp_indirect
assert not vndk_sp_unused & vndk_sp_indirect_unused
# Define helper functions for vndk_sp sets.
def is_vndk_sp_public(lib):
@@ -2617,13 +2625,12 @@ class GenericRefs(object):
def _load_from_sym_dir(self, root):
root = os.path.abspath(root)
prefix_len = len(root) + 1
for base, dirnames, filenames in os.walk(root):
for base, _, filenames in os.walk(root):
for filename in filenames:
if not filename.endswith('.sym'):
continue
path = os.path.join(base, filename)
lib_path = '/' + path[prefix_len:-4]
with open(path, 'r') as f:
self.add(lib_path, ELF.load_dump(path))
@staticmethod
@@ -2694,7 +2701,7 @@ def _build_lib_names_dict(graph, min_name_len=6, lib_ext='.so'):
def _enumerate_partition_paths(partition, root):
prefix_len = len(root) + 1
for base, dirs, files in os.walk(root):
for base, _, files in os.walk(root):
for filename in files:
path = os.path.join(base, filename)
if not is_accessible(path):
@@ -2786,7 +2793,7 @@ class ModuleInfo(object):
mods = collections.defaultdict(set)
installed_path_patt = re.compile(
'.*[\\\\/]target[\\\\/]product[\\\\/][^\\\\/]+([\\\\/].*)$')
for name, module in json.items():
for module in json.values():
for path in module['installed']:
match = installed_path_patt.match(path)
if match:
@@ -2922,15 +2929,15 @@ class ELFGraphCommand(Command):
if args.load_generic_refs:
return GenericRefs.create_from_sym_dir(args.load_generic_refs)
if args.aosp_system:
return GenericRefs.create_from_image_dir(args.aosp_system,
'/system')
return GenericRefs.create_from_image_dir(
args.aosp_system, '/system')
return None
def _check_arg_dir_exists(self, arg_name, dirs):
for path in dirs:
if not os.path.exists(path):
print('error: Failed to find the directory "{}" specified in {}'
.format(path, arg_name), file=sys.stderr)
print('error: Failed to find the directory "{}" specified in '
'"{}"'.format(path, arg_name), file=sys.stderr)
sys.exit(1)
if not os.path.isdir(path):
print('error: Path "{}" specified in {} is not a directory'
@@ -2970,7 +2977,8 @@ class VNDKCommandBase(ELFGraphCommand):
def add_argparser_options(self, parser):
super(VNDKCommandBase, self).add_argparser_options(parser)
parser.add_argument('--no-default-dlopen-deps', action='store_true',
parser.add_argument(
'--no-default-dlopen-deps', action='store_true',
help='do not add default dlopen dependencies')
parser.add_argument(
@@ -2987,13 +2995,12 @@ class VNDKCommandBase(ELFGraphCommand):
"""Create all essential data structures for VNDK computation."""
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
super(VNDKCommandBase, self).\
create_from_args(args)
super(VNDKCommandBase, self).create_from_args(args)
if not args.no_default_dlopen_deps:
script_dir = os.path.dirname(os.path.abspath(__file__))
minimum_dlopen_deps = os.path.join(script_dir, 'datasets',
'minimum_dlopen_deps.txt')
minimum_dlopen_deps = os.path.join(
script_dir, 'datasets', 'minimum_dlopen_deps.txt')
graph.add_dlopen_deps(minimum_dlopen_deps)
return (generic_refs, graph, tagged_paths, vndk_lib_dirs)
@@ -3078,9 +3085,7 @@ class VNDKCommand(VNDKCommandBase):
def _print_make(self, vndk_lib, file=sys.stdout):
def get_module_name(path):
name = os.path.basename(path)
root, ext = os.path.splitext(name)
return root
return os.path.splitext(os.path.basename(path))[0]
def get_module_names(lib_set):
return sorted({get_module_name(lib.path) for lib in lib_set})
@@ -3122,7 +3127,8 @@ class VNDKCommand(VNDKCommandBase):
# Print the file size of all ELF files.
for lib in sorted(graph.all_libs()):
writer.writerow((lib.path, collect_tags(lib), lib.elf.file_size,
writer.writerow((
lib.path, collect_tags(lib), lib.elf.file_size,
lib.elf.ro_seg_file_size, lib.elf.ro_seg_mem_size,
lib.elf.rw_seg_file_size, lib.elf.rw_seg_mem_size))
@@ -3171,8 +3177,7 @@ class VNDKCommand(VNDKCommandBase):
writer.writerow(['Total', None] + calc_total_size(graph.all_libs()))
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
generic_refs, graph, tagged_paths, _ = self.create_from_args(args)
if args.warn_incorrect_partition:
self._warn_incorrect_partition(graph)
@@ -3205,8 +3210,7 @@ class DepsInsightCommand(VNDKCommandBase):
parser.add_argument('--module-info')
parser.add_argument(
'--output', '-o', help='output directory')
parser.add_argument('--output', '-o', help='output directory')
@staticmethod
def serialize_data(libs, vndk_lib, module_info):
@@ -3274,8 +3278,7 @@ class DepsInsightCommand(VNDKCommandBase):
return (strs, mods)
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
generic_refs, graph, tagged_paths, _ = self.create_from_args(args)
module_info = ModuleInfo.load_from_path_or_default(args.module_info)
@@ -3285,14 +3288,15 @@ class DepsInsightCommand(VNDKCommandBase):
args.action_ineligible_vndk)
# Serialize data.
strs, mods = self.serialize_data(list(graph.all_libs()), vndk_lib,
module_info)
strs, mods = self.serialize_data(
list(graph.all_libs()), vndk_lib, module_info)
# Generate output files.
makedirs(args.output, exist_ok=True)
script_dir = os.path.dirname(os.path.abspath(__file__))
for name in ('index.html', 'insight.css', 'insight.js'):
shutil.copyfile(os.path.join(script_dir, 'assets', 'insight', name),
shutil.copyfile(
os.path.join(script_dir, 'assets', 'insight', name),
os.path.join(args.output, name))
with open(os.path.join(args.output, 'insight-data.js'), 'w') as f:
@@ -3332,12 +3336,14 @@ class DepsCommand(ELFGraphCommand):
parser.add_argument('--module-info')
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
_, graph, _, _ = self.create_from_args(args)
module_info = ModuleInfo.load_from_path_or_default(args.module_info)
path_filter = re.compile(args.path_filter) if args.path_filter else None
if args.path_filter:
path_filter = re.compile(args.path_filter)
else:
path_filter = None
if args.symbols:
def collect_symbols(user, definer):
@@ -3421,8 +3427,7 @@ class DepsClosureCommand(ELFGraphCommand):
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
_, graph, _, _ = self.create_from_args(args)
# Find root/excluded libraries by their paths.
def report_error(path):
@@ -3476,8 +3481,7 @@ class DepsUnresolvedCommand(ELFGraphCommand):
print('\tUNRESOLVED_SYMBOL:', symbol)
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
_, graph, _, _ = self.create_from_args(args)
module_info = ModuleInfo.load_from_path_or_default(args.module_info)
libs = graph.all_libs()
@@ -3496,12 +3500,8 @@ class ApkDepsCommand(ELFGraphCommand):
super(ApkDepsCommand, self).__init__(
'apk-deps', help='Print APK dependencies for debugging')
def add_argparser_options(self, parser):
super(ApkDepsCommand, self).add_argparser_options(parser)
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
_, graph, _, _ = self.create_from_args(args)
apk_deps = scan_apk_dep(graph, args.system, args.vendor)
@@ -3622,8 +3622,8 @@ class CheckDepCommand(CheckDepCommandBase):
.format(lib.path, dep.path),
file=sys.stderr)
elif dep_properties.vndk:
print('error: vendor lib "{}" depends on vndk "{}" but '
'it must be copied to /system/lib[64]/vndk.'
print('error: vendor lib "{}" depends on vndk "{}" '
'but it must be copied to /system/lib[64]/vndk.'
.format(lib.path, dep.path),
file=sys.stderr)
else:
@@ -3638,7 +3638,7 @@ class CheckDepCommand(CheckDepCommandBase):
return num_errors
def _check_dt_needed_ordering(self, graph, module_info):
def _check_dt_needed_ordering(self, graph):
"""Check DT_NEEDED entries order of all libraries"""
num_errors = 0
@@ -3710,7 +3710,7 @@ class CheckDepCommand(CheckDepCommandBase):
module_info)
if args.check_dt_needed_ordering:
num_errors += self._check_dt_needed_ordering(graph, module_info)
num_errors += self._check_dt_needed_ordering(graph)
if args.check_apk:
num_errors += self._check_apk_dep(graph, args.system, args.vendor,
@@ -3740,67 +3740,6 @@ class DumpDexStringCommand(Command):
print(repr(string))
class CheckEligibleListCommand(CheckDepCommandBase):
def __init__(self):
super(CheckEligibleListCommand, self).__init__(
'check-eligible-list', help='Check the eligible list')
def _check_eligible_vndk_dep(self, graph, tagged_libs, module_info):
"""Check whether eligible sets are self-contained."""
num_errors = 0
indirect_libs = (tagged_libs.ll_ndk_indirect |
tagged_libs.vndk_sp_indirect_private |
tagged_libs.fwk_only_rs)
eligible_libs = (tagged_libs.ll_ndk | tagged_libs.vndk_sp |
tagged_libs.vndk_sp_indirect | tagged_libs.vndk)
# Check eligible vndk is self-contained.
for lib in sorted(eligible_libs):
bad_deps = []
for dep in lib.deps_all:
if dep not in eligible_libs and dep not in indirect_libs:
print('error: eligible lib "{}" should not depend on '
'non-eligible lib "{}".'.format(lib.path, dep.path),
file=sys.stderr)
bad_deps.append(dep)
num_errors += 1
if bad_deps:
self._dump_dep(lib, bad_deps, module_info)
# Check the libbinder dependencies.
for lib in sorted(eligible_libs):
bad_deps = []
for dep in lib.deps_all:
if os.path.basename(dep.path) == 'libbinder.so':
print('error: eligible lib "{}" should not depend on '
'libbinder.so.'.format(lib.path), file=sys.stderr)
bad_deps.append(dep)
num_errors += 1
if bad_deps:
self._dump_dep(lib, bad_deps, module_info)
return num_errors
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
tagged_paths = TaggedPathDict.create_from_csv_path(
args.tag_file, vndk_lib_dirs)
tagged_libs = TaggedLibDict.create_from_graph(
graph, tagged_paths, generic_refs)
module_info = ModuleInfo.load_from_path_or_default(args.module_info)
num_errors = self._check_eligible_vndk_dep(graph, tagged_libs,
module_info)
return 0 if num_errors == 0 else 1
class DepGraphCommand(ELFGraphCommand):
def __init__(self):
super(DepGraphCommand, self).__init__(
@@ -3848,12 +3787,13 @@ class DepGraphCommand(ELFGraphCommand):
'violates': [],
}
for dep in lib.deps_all:
if self._check_if_allowed(tag,
self._get_tag_from_lib(dep, tagged_paths)):
if self._check_if_allowed(
tag, self._get_tag_from_lib(dep, tagged_paths)):
lib_item['depends'].append(dep.path)
else:
lib_item['violates'].append([dep.path, lib.get_dep_linked_symbols(dep)])
violate_count += 1;
lib_item['violates'].append([
dep.path, lib.get_dep_linked_symbols(dep)])
violate_count += 1
lib_item['violate_count'] = violate_count
if violate_count > 0:
if not tag in violate_libs:
@@ -3863,8 +3803,7 @@ class DepGraphCommand(ELFGraphCommand):
return data, violate_libs
def main(self, args):
generic_refs, graph, tagged_paths, vndk_lib_dirs = \
self.create_from_args(args)
_, graph, tagged_paths, vndk_lib_dirs = self.create_from_args(args)
tagged_paths = TaggedPathDict.create_from_csv_path(
args.tag_file, vndk_lib_dirs)
@@ -3880,8 +3819,8 @@ class DepGraphCommand(ELFGraphCommand):
shutil.copyfile(os.path.join(script_dir, 'assets', 'visual', name),
os.path.join(args.output, name))
with open(os.path.join(args.output, 'dep-data.js'), 'w') as f:
f.write('var violatedLibs = ' + json.dumps(violate_libs) +
'\nvar depData = ' + json.dumps(data) + ';')
f.write('var violatedLibs = ' + json.dumps(violate_libs) + ';\n')
f.write('var depData = ' + json.dumps(data) + ';\n')
return 0
@@ -3905,7 +3844,6 @@ def main():
register_subcmd(DepsUnresolvedCommand())
register_subcmd(ApkDepsCommand())
register_subcmd(CheckDepCommand())
register_subcmd(CheckEligibleListCommand())
register_subcmd(DepGraphCommand())
register_subcmd(DumpDexStringCommand())