Merge changes Ifed15e60,Icff8aeeb,I0365be01,I746b07a4 am: 0092ef8928

am: 369e7dd7fa

Change-Id: I3e0c1a5ef4c703f1fd242cbf0d5b28c41fefc632
This commit is contained in:
Logan Chien
2017-03-08 02:00:45 +00:00
committed by android-build-merger
8 changed files with 376 additions and 105 deletions

View File

@@ -6,6 +6,7 @@ import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import tempfile
import unittest
from compat import StringIO
@@ -39,6 +40,24 @@ class ElfSymTest(unittest.TestCase):
class ELFTest(unittest.TestCase):
def test_get_ei_class_from_name(self):
self.assertEqual(ELF.ELFCLASS32, ELF.get_ei_class_from_name('32'))
self.assertEqual(ELF.ELFCLASS64, ELF.get_ei_class_from_name('64'))
def test_get_ei_data_from_name(self):
self.assertEqual(ELF.ELFDATA2LSB,
ELF.get_ei_data_from_name('Little-Endian'))
self.assertEqual(ELF.ELFDATA2MSB,
ELF.get_ei_data_from_name('Big-Endian'))
def test_get_e_machine_from_name(self):
self.assertEqual(0, ELF.get_e_machine_from_name('EM_NONE'))
self.assertEqual(3, ELF.get_e_machine_from_name('EM_386'))
self.assertEqual(8, ELF.get_e_machine_from_name('EM_MIPS'))
self.assertEqual(40, ELF.get_e_machine_from_name('EM_ARM'))
self.assertEqual(62, ELF.get_e_machine_from_name('EM_X86_64'))
self.assertEqual(183, ELF.get_e_machine_from_name('EM_AARCH64'))
def test_repr(self):
elf = ELF()
self.assertEqual(elf, eval(repr(elf)))
@@ -106,15 +125,44 @@ class ELFTest(unittest.TestCase):
'IMP_SYMBOL\te\n',
actual_output)
def test_dump_exported_symbols(self):
elf = ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB, 183, ['a'], ['b'],
['libc.so', 'libm.so'], {'hello', 'world'})
def test_parse_dump_file(self):
data = ('EI_CLASS\t64\n'
'EI_DATA\t\tLittle-Endian\n'
'E_MACHINE\tEM_AARCH64\n'
'DT_RPATH\trpath_1\n'
'DT_RPATH\trpath_2\n'
'DT_RUNPATH\trunpath_1\n'
'DT_RUNPATH\trunpath_2\n'
'DT_NEEDED\tlibc.so\n'
'DT_NEEDED\tlibm.so\n'
'EXP_SYMBOL\texported_1\n'
'EXP_SYMBOL\texported_2\n'
'IMP_SYMBOL\timported_1\n'
'IMP_SYMBOL\timported_2\n')
f = StringIO()
elf.dump_exported_symbols(f)
actual_output = f.getvalue()
def check_parse_dump_file_result(res):
self.assertEqual(ELF.ELFCLASS64, res.ei_class)
self.assertEqual(ELF.ELFDATA2LSB, res.ei_data)
self.assertEqual(183, res.e_machine)
self.assertEqual(['rpath_1', 'rpath_2'], res.dt_rpath)
self.assertEqual(['runpath_1', 'runpath_2'], res.dt_runpath)
self.assertEqual(['libc.so', 'libm.so'], res.dt_needed)
self.assertSetEqual({'exported_1', 'exported_2'},
res.exported_symbols)
self.assertSetEqual({'imported_1', 'imported_2'},
res.imported_symbols)
# Parse ELF dump from the string buffer.
check_parse_dump_file_result(ELF.load_dumps(data))
# Parse ELF dump from the given file path.
with tempfile.NamedTemporaryFile('w+') as f:
f.write(data)
f.flush()
f.seek(0)
check_parse_dump_file_result(ELF.load_dump(f.name))
self.assertEqual('hello\nworld\n', actual_output)
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,88 @@
#!/usr/bin/env python3
from __future__ import print_function
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import unittest
from vndk_definition_tool import ELFLinkData, PT_SYSTEM, PT_VENDOR
class ELFLinkDataTest(unittest.TestCase):
def setUp(self):
self.x = ELFLinkData(PT_SYSTEM, '/system/lib/libx.so', None)
self.y = ELFLinkData(PT_SYSTEM, '/system/lib/liby.so', None)
self.z = ELFLinkData(PT_SYSTEM, '/system/lib/libz.so', None)
self.w = ELFLinkData(PT_SYSTEM, '/system/lib/libw.so', None)
self.v = ELFLinkData(PT_VENDOR, '/vendor/lib/libv.so', None)
self.x.add_dep(self.y, ELFLinkData.NEEDED)
self.x.add_dep(self.z, ELFLinkData.DLOPEN)
self.z.add_dep(self.w, ELFLinkData.NEEDED)
self.z.add_dep(self.w, ELFLinkData.DLOPEN)
def test_add_dep_and_accessors(self):
self.assertIn(self.y, self.x.dt_deps)
self.assertIn(self.x, self.y.dt_users)
self.assertNotIn(self.y, self.x.dl_deps)
self.assertNotIn(self.x, self.y.dl_users)
self.assertIn(self.z, self.x.dl_deps)
self.assertIn(self.x, self.z.dl_users)
self.assertNotIn(self.z, self.x.dt_deps)
self.assertNotIn(self.x, self.z.dt_users)
def test_remove_dep(self):
self.assertIn(self.y, self.x.dt_deps)
self.assertIn(self.x, self.y.dt_users)
with self.assertRaises(KeyError):
self.x.remove_dep(self.y, ELFLinkData.DLOPEN)
self.assertIn(self.y, self.x.dt_deps)
self.assertIn(self.x, self.y.dt_users)
self.x.remove_dep(self.y, ELFLinkData.NEEDED)
self.assertNotIn(self.y, self.x.dt_deps)
self.assertNotIn(self.x, self.y.dt_users)
def test_num_deps(self):
self.assertEqual(2, self.x.num_deps)
self.assertEqual(0, self.y.num_deps)
self.assertEqual(0, self.w.num_deps)
self.assertEqual(0, self.v.num_deps)
# NEEDED and DLOPEN are counted twice.
self.assertEqual(2, self.z.num_deps)
def test_num_users(self):
self.assertEqual(0, self.x.num_users)
self.assertEqual(1, self.y.num_users)
self.assertEqual(1, self.z.num_users)
self.assertEqual(0, self.v.num_users)
# NEEDED and DLOPEN are counted twice.
self.assertEqual(2, self.w.num_users)
def test_has_dep(self):
self.assertTrue(self.x.has_dep(self.y))
self.assertTrue(self.x.has_dep(self.z))
self.assertFalse(self.x.has_dep(self.x))
self.assertFalse(self.x.has_dep(self.w))
def test_has_user(self):
self.assertTrue(self.y.has_user(self.x))
self.assertTrue(self.z.has_user(self.x))
self.assertFalse(self.x.has_user(self.x))
self.assertFalse(self.w.has_user(self.x))
def test_is_system_lib(self):
self.assertTrue(self.x.is_system_lib())
self.assertFalse(self.v.is_system_lib())
if __name__ == '__main__':
unittest.main()

View File

@@ -13,7 +13,7 @@ from compat import TemporaryDirectory, makedirs
from vndk_definition_tool import GenericRefs
test_dir_base = None
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
class MockELF(object):
@@ -28,62 +28,39 @@ class MockLib(object):
class GenericRefsTest(unittest.TestCase):
def _build_file_fixture(self, path, content):
makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as f:
f.write(content)
def _build_dir_fixtures(self, test_dir):
lib32 = os.path.join(test_dir, 'system', 'lib')
lib64 = os.path.join(test_dir, 'system', 'lib64')
for lib_dir in (lib32, lib64):
self._build_file_fixture(os.path.join(lib_dir, 'libc.so.sym'),
'fclose\nfopen\nfread\nfwrite\n')
self._build_file_fixture(os.path.join(lib_dir, 'libm.so.sym'),
'cos\nsin\ntan\n')
def _build_fixture(self):
res = GenericRefs()
res.add('/system/lib/libc.so', {'fclose', 'fopen', 'fread', 'fwrite'})
res.add('/system/lib/libm.so', {'cos', 'sin', 'tan'})
res.add('/system/lib64/libc.so', {'fclose', 'fopen', 'fread', 'fwrite'})
res.add('/system/lib64/libm.so', {'cos', 'sin', 'tan'})
return res
def setUp(self):
self.ref = GenericRefs()
self.ref.add('/system/lib/libc.so',
MockELF({'fclose', 'fopen', 'fread', 'fwrite'}))
self.ref.add('/system/lib/libm.so',
MockELF({'cos', 'sin', 'tan'}))
self.ref.add('/system/lib64/libc.so',
MockELF({'fclose', 'fopen', 'fread', 'fwrite'}))
self.ref.add('/system/lib64/libm.so',
MockELF({'cos', 'sin', 'tan'}))
def test_create_from_dir(self):
try:
if test_dir_base:
test_dir = test_dir_base
else:
tmp_dir = TemporaryDirectory()
test_dir = tmp_dir.name
input_dir = os.path.join(SCRIPT_DIR, 'testdata', 'test_generic_refs')
self._build_dir_fixtures(test_dir)
g = GenericRefs.create_from_dir(test_dir)
self.assertEqual(4, len(g.refs))
g = GenericRefs.create_from_dir(input_dir)
self.assertEqual(4, len(g.refs))
self.assertIn('/system/lib/libc.so', g.refs)
self.assertIn('/system/lib/libm.so', g.refs)
self.assertIn('/system/lib64/libc.so', g.refs)
self.assertIn('/system/lib64/libm.so', g.refs)
self.assertIn('/system/lib/libc.so', g.refs)
self.assertIn('/system/lib/libm.so', g.refs)
self.assertIn('/system/lib64/libc.so', g.refs)
self.assertIn('/system/lib64/libm.so', g.refs)
self.assertEqual({'fclose', 'fopen', 'fread', 'fwrite'},
g.refs['/system/lib/libc.so'])
self.assertEqual({'fclose', 'fopen', 'fread', 'fwrite'},
g.refs['/system/lib64/libc.so'])
self.assertEqual({'fclose', 'fopen', 'fread', 'fwrite'},
g.refs['/system/lib/libc.so'].exported_symbols)
self.assertEqual({'fclose', 'fopen', 'fread', 'fwrite'},
g.refs['/system/lib64/libc.so'].exported_symbols)
self.assertEqual({'cos', 'sin', 'tan'},
g.refs['/system/lib/libm.so'])
self.assertEqual({'cos', 'sin', 'tan'},
g.refs['/system/lib64/libm.so'])
finally:
if not test_dir_base:
tmp_dir.cleanup()
self.assertEqual({'cos', 'sin', 'tan'},
g.refs['/system/lib/libm.so'].exported_symbols)
self.assertEqual({'cos', 'sin', 'tan'},
g.refs['/system/lib64/libm.so'].exported_symbols)
def test_classify_lib(self):
g = self._build_fixture()
libc_sub = MockLib('/system/lib/libc.so', {'fclose', 'fopen', 'fread'})
libc_sup = MockLib('/system/lib/libc.so',
{'fclose', 'fopen', 'fread', 'fwrite', 'open'})
@@ -91,40 +68,25 @@ class GenericRefsTest(unittest.TestCase):
{'fclose', 'fopen', 'fread', 'fwrite'})
libfoo = MockLib('/system/lib/libfoo.so', {})
self.assertEqual(GenericRefs.MODIFIED, g.classify_lib(libc_sub))
self.assertEqual(GenericRefs.EXPORT_SUPER_SET, g.classify_lib(libc_sup))
self.assertEqual(GenericRefs.EXPORT_EQUAL, g.classify_lib(libc_eq))
self.assertEqual(GenericRefs.NEW_LIB, g.classify_lib(libfoo))
self.assertEqual(GenericRefs.MODIFIED, self.ref.classify_lib(libc_sub))
self.assertEqual(GenericRefs.EXPORT_SUPER_SET,
self.ref.classify_lib(libc_sup))
self.assertEqual(GenericRefs.EXPORT_EQUAL,
self.ref.classify_lib(libc_eq))
self.assertEqual(GenericRefs.NEW_LIB, self.ref.classify_lib(libfoo))
def test_is_equivalent_lib(self):
g = self._build_fixture()
libc_sub = MockLib('/system/lib/libc.so', {'fclose', 'fopen', 'fread'})
libc_sup = MockLib('/system/lib/libc.so',
{'fclose', 'fopen', 'fread', 'fwrite', 'open'})
libc_eq = MockLib('/system/lib/libc.so',
{'fclose', 'fopen', 'fread', 'fwrite'})
self.assertFalse(g.is_equivalent_lib(libc_sub))
self.assertFalse(g.is_equivalent_lib(libc_sup))
self.assertFalse(self.ref.is_equivalent_lib(libc_sub))
self.assertFalse(self.ref.is_equivalent_lib(libc_sup))
self.assertTrue(g.is_equivalent_lib(libc_eq))
self.assertTrue(self.ref.is_equivalent_lib(libc_eq))
def main():
# Parse command line arguments.
parser = argparse.ArgumentParser()
parser.add_argument('--test-dir', help='directory for temporary files')
args, unittest_args = parser.parse_known_args()
# Convert command line options.
global test_dir_base
if args.test_dir:
test_dir_base = args.test_dir
# Run unit test.
unittest.main(argv=[sys.argv[0]] + unittest_args)
if __name__ == '__main__':
main()
unittest.main()

View File

@@ -0,0 +1,7 @@
EI_CLASS 32
EI_DATA Little-Endian
E_MACHINE EM_ARM
EXP_SYMBOL fclose
EXP_SYMBOL fopen
EXP_SYMBOL fread
EXP_SYMBOL fwrite

View File

@@ -0,0 +1,6 @@
EI_CLASS 32
EI_DATA Little-Endian
E_MACHINE EM_ARM
EXP_SYMBOL cos
EXP_SYMBOL sin
EXP_SYMBOL tan

View File

@@ -0,0 +1,7 @@
EI_CLASS 64
EI_DATA Little-Endian
E_MACHINE EM_AARCH64
EXP_SYMBOL fclose
EXP_SYMBOL fopen
EXP_SYMBOL fread
EXP_SYMBOL fwrite

View File

@@ -0,0 +1,6 @@
EI_CLASS 64
EI_DATA Little-Endian
E_MACHINE EM_AARCH64
EXP_SYMBOL cos
EXP_SYMBOL sin
EXP_SYMBOL tan

View File

@@ -4,6 +4,7 @@ from __future__ import print_function
import argparse
import collections
import itertools
import os
import re
import stat
@@ -140,6 +141,26 @@ class ELF(object):
}
@staticmethod
def _dict_find_key_by_value(d, dst):
for key, value in d.items():
if value == dst:
return key
raise KeyError(dst)
@staticmethod
def get_ei_class_from_name(name):
return ELF._dict_find_key_by_value(ELF._ELF_CLASS_NAMES, name)
@staticmethod
def get_ei_data_from_name(name):
return ELF._dict_find_key_by_value(ELF._ELF_DATA_NAMES, name)
@staticmethod
def get_e_machine_from_name(name):
return ELF._dict_find_key_by_value(ELF._ELF_MACHINE_IDS, name)
__slots__ = ('ei_class', 'ei_data', 'e_machine', 'dt_rpath', 'dt_runpath',
'dt_needed', 'exported_symbols', 'imported_symbols',)
@@ -211,12 +232,6 @@ class ELF(object):
for symbol in self.sorted_imported_symbols:
print('IMP_SYMBOL\t' + symbol, file=file)
def dump_exported_symbols(self, file=None):
"""Print exported symbols to the file"""
file = file if file is not None else sys.stdout
for symbol in self.sorted_exported_symbols:
print(symbol, file=file)
# Extract zero-terminated buffer slice.
def _extract_zero_terminated_buf_slice(self, buf, offset):
"""Extract a zero-terminated buffer slice from the given offset"""
@@ -387,6 +402,47 @@ class ELF(object):
with mmap(f.fileno(), st.st_size, access=ACCESS_READ) as image:
self._parse_from_buf(image)
def _parse_from_dump_lines(self, path, lines):
patt = re.compile('^([A-Za-z_]+)\t+(.*)$')
for line_no, line in enumerate(lines):
match = patt.match(line)
if not match:
print('error: {}: {}: failed to parse'
.format(path, line_no + 1), file=sys.stderr)
continue
key = match.group(1)
value = match.group(2)
if key == 'EI_CLASS':
self.ei_class = ELF.get_ei_class_from_name(value)
elif key == 'EI_DATA':
self.ei_data = ELF.get_ei_data_from_name(value)
elif key == 'E_MACHINE':
self.e_machine = ELF.get_e_machine_from_name(value)
elif key == 'DT_RPATH':
self.dt_rpath.append(intern(value))
elif key == 'DT_RUNPATH':
self.dt_runpath.append(intern(value))
elif key == 'DT_NEEDED':
self.dt_needed.append(intern(value))
elif key == 'EXP_SYMBOL':
self.exported_symbols.add(intern(value))
elif key == 'IMP_SYMBOL':
self.imported_symbols.add(intern(value))
else:
print('error: {}: {}: unknown tag name: {}'
.format(path, line_no + 1, key), file=sys.stderr)
def _parse_from_dump_file(self, path):
"""Load information from ELF dump file."""
with open(path, 'r') as f:
self._parse_from_dump_lines(path, f)
def _parse_from_dump_buf(self, buf):
"""Load information from ELF dump buffer."""
self._parse_from_dump_lines('<str:0x{:x}>'.format(id(buf)),
buf.splitlines())
@staticmethod
def load(path):
"""Create an ELF instance from the file path"""
@@ -401,6 +457,20 @@ class ELF(object):
elf._parse_from_buf(buf)
return elf
@staticmethod
def load_dump(path):
"""Create an ELF instance from a dump file path"""
elf = ELF()
elf._parse_from_dump_file(path)
return elf
@staticmethod
def load_dumps(buf):
"""Create an ELF instance from a dump file buffer"""
elf = ELF()
elf._parse_from_dump_buf(buf)
return elf
#------------------------------------------------------------------------------
# NDK and Banned Libraries
@@ -539,19 +609,71 @@ class ELFResolver(object):
class ELFLinkData(object):
NEEDED = 0 # Dependencies recorded in DT_NEEDED entries.
DLOPEN = 1 # Dependencies introduced by dlopen().
def __init__(self, partition, path, elf):
self.partition = partition
self.path = path
self.elf = elf
self.deps = set()
self.users = set()
self._deps = (set(), set())
self._users = (set(), set())
self.is_ndk = NDK_LIBS.is_ndk(path)
self.unresolved_symbols = set()
self.linked_symbols = dict()
def add_dep(self, dst):
self.deps.add(dst)
dst.users.add(self)
def add_dep(self, dst, ty):
self._deps[ty].add(dst)
dst._users[ty].add(self)
def remove_dep(self, dst, ty):
self._deps[ty].remove(dst)
dst._users[ty].remove(self)
@property
def num_deps(self):
"""Get the number of dependencies. If a library is linked by both
NEEDED and DLOPEN relationship, then it will be counted twice."""
return sum(len(deps) for deps in self._deps)
@property
def deps(self):
return itertools.chain.from_iterable(self._deps)
@property
def dt_deps(self):
return self._deps[self.NEEDED]
@property
def dl_deps(self):
return self._deps[self.DLOPEN]
@property
def num_users(self):
"""Get the number of users. If a library is linked by both NEEDED and
DLOPEN relationship, then it will be counted twice."""
return sum(len(users) for users in self._users)
@property
def users(self):
return itertools.chain.from_iterable(self._users)
@property
def dt_users(self):
return self._users[self.NEEDED]
@property
def dl_users(self):
return self._users[self.DLOPEN]
def has_dep(self, dst):
return any(dst in deps for deps in self._deps)
def has_user(self, dst):
return any(dst in users for users in self._users)
def is_system_lib(self):
return self.partition == PT_SYSTEM
def sorted_lib_path_list(libs):
@@ -575,12 +697,15 @@ class ELFLinker(object):
self.lib_pt[partition][path] = node
return node
def add_dep(self, src_path, dst_path):
def add_dep(self, src_path, dst_path, ty):
for lib_set in (self.lib32, self.lib64):
src = lib_set.get(src_path)
dst = lib_set.get(dst_path)
if src and dst:
src.add_dep(dst)
src.add_dep(dst, ty)
return
print('error: cannot add dependency from {} to {}.'
.format(src_path, dst_path), file=sys.stderr)
def map_path_to_lib(self, path):
for lib_set in (self.lib32, self.lib64):
@@ -631,7 +756,8 @@ class ELFLinker(object):
for line in f:
match = patt.match(line)
if match:
self.add_dep(match.group(1), match.group(2))
self.add_dep(match.group(1), match.group(2),
ELFLinkData.DLOPEN)
def _find_exported_symbol(self, symbol, libs):
"""Find the shared library with the exported symbol."""
@@ -660,7 +786,7 @@ class ELFLinker(object):
print('warning: {}: Missing needed library: {} Tried: {}'
.format(lib.path, dt_needed, candidates), file=sys.stderr)
continue
lib.add_dep(dep)
lib.add_dep(dep, ELFLinkData.NEEDED)
imported_libs.append(dep)
return imported_libs
@@ -681,6 +807,27 @@ class ELFLinker(object):
self.lib64,
ELFResolver(self.lib64, ['/system/lib64', '/vendor/lib64']))
def _resolve_lib_extended_symbol_users(self, generic_refs, lib):
"""Resolve the users of the extended exported symbols of a library."""
try:
ref_lib = generic_refs.refs[lib.path]
except KeyError:
lib.extended_symbol_users = lib.users
return
for user in lib.users:
for symbol, imp_lib in user.linked_symbols.items():
if imp_lib is not lib:
continue
if symbol not in ref_lib.exported_symbols:
lib.extended_symbol_users.add(user)
def resolve_extended_symbol_users(self, generic_refs):
"""Resolve the users of the extended exported symbols."""
for lib_set in self.lib_pt:
for lib in lib_set.values():
self._resolve_lib_extended_symbol_users(generic_refs, lib)
def compute_matched_libs(self, path_patterns, closure=False,
is_excluded_libs=None):
patt = re.compile('|'.join('(?:' + p + ')' for p in path_patterns))
@@ -872,8 +1019,8 @@ class GenericRefs(object):
def __init__(self):
self.refs = dict()
def add(self, name, symbols):
self.refs[name] = symbols
def add(self, name, elf):
self.refs[name] = elf
def _load_from_dir(self, root):
root = os.path.abspath(root)
@@ -885,7 +1032,7 @@ class GenericRefs(object):
path = os.path.join(base, filename)
lib_name = '/' + path[prefix_len:-4]
with open(path, 'r') as f:
self.add(lib_name, set(line.strip() for line in f))
self.add(lib_name, ELF.load_dump(path))
@staticmethod
def create_from_dir(root):
@@ -894,13 +1041,13 @@ class GenericRefs(object):
return result
def classify_lib(self, lib):
ref_lib_symbols = self.refs.get(lib.path)
if not ref_lib_symbols:
ref_lib = self.refs.get(lib.path)
if not ref_lib:
return GenericRefs.NEW_LIB
exported_symbols = lib.elf.exported_symbols
if exported_symbols == ref_lib_symbols:
if exported_symbols == ref_lib.exported_symbols:
return GenericRefs.EXPORT_EQUAL
if exported_symbols > ref_lib_symbols:
if exported_symbols > ref_lib.exported_symbols:
return GenericRefs.EXPORT_SUPER_SET
return GenericRefs.MODIFIED
@@ -966,7 +1113,7 @@ class CreateGenericRefCommand(Command):
out = os.path.join(args.output, name) + '.sym'
makedirs(os.path.dirname(out), exist_ok=True)
with open(out, 'w') as f:
elf.dump_exported_symbols(f)
elf.dump(f)
except ELFError:
pass
return 0
@@ -1025,7 +1172,7 @@ class VNDKCommand(ELFGraphCommand):
def _warn_incorrect_partition_lib_set(self, lib_set, partition, error_msg):
for lib in lib_set.values():
if not lib.users:
if not lib.num_users:
continue
if all((user.partition != partition for user in lib.users)):
print(error_msg.format(lib.path), file=sys.stderr)