LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "def-tool: Add APEX module support"

Browse Source
This commit is contained in:
Logan Chien
2019-04-03 03:08:46 +00:00
committed by Gerrit Code Review
parent 7f8b899fe2 443fa6142d
commit a9c00011f3
8 changed files with 371 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
vndk/tools/definition-tool/tests/test_elf_linker.py
View File

@@ -4,7 +4,7 @@ import re
import tempfile
from vndk_definition_tool import (
ELF, GenericRefs, PT_SYSTEM, PT_VENDOR, VNDKLibDir)
ELF, ELFLinker, GenericRefs, PT_SYSTEM, PT_VENDOR, VNDKLibDir)
from .compat import StringIO, TestCase, patch
from .utils import GraphBuilder
@@ -460,6 +460,51 @@ class ELFLinkerTest(TestCase):
self.assertIn(libvndk_sp_d_64, libvndk_sp_c_64.deps_all)
def test_rewrite_apex_modules(self):
graph = ELFLinker()
libfoo = graph.add_lib(PT_SYSTEM, '/system/apex/foo/lib/libfoo.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
libbar = graph.add_lib(PT_SYSTEM, '/system/apex/bar/lib/libbar.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
graph.rewrite_apex_modules()
self.assertEqual(libfoo.path, '/apex/foo/lib/libfoo.so')
self.assertEqual(libbar.path, '/apex/bar/lib/libbar.so')
def test_link_apex_modules(self):
graph = ELFLinker()
libfoo = graph.add_lib(PT_SYSTEM, '/system/apex/foo/lib/libfoo.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
libbar = graph.add_lib(PT_SYSTEM, '/system/lib/libbar.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB,
dt_needed=['libfoo.so']))
graph.rewrite_apex_modules()
graph.resolve_deps()
self.assertIn(libfoo, libbar.deps_all)
def test_link_apex_bionic(self):
graph = ELFLinker()
libc = graph.add_lib(
PT_SYSTEM, '/system/apex/com.android.runtime/lib/bionic/libc.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
libbar = graph.add_lib(
PT_SYSTEM, '/system/lib/libbar.so',
ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB, dt_needed=['libc.so']))
graph.rewrite_apex_modules()
graph.resolve_deps()
self.assertIn(libc, libbar.deps_all)
class ELFLinkerDlopenDepsTest(TestCase):
def test_add_dlopen_deps(self):
gb = GraphBuilder()

49
vndk/tools/definition-tool/tests/test_scandir.py Normal file
View File

@@ -0,0 +1,49 @@
#!/usr/bin/env python3
import os
import unittest
from vndk_definition_tool import scandir
class ScanDirTest(unittest.TestCase):
def test_scandir(self):
testdata_dir = os.path.join(
os.path.dirname(__file__), 'testdata', 'test_scandir')
num_entries = 0
for ent in scandir(testdata_dir):
num_entries += 1
self.assertEqual(ent.path, os.path.join(testdata_dir, ent.name))
if ent.name == 'test_file':
self.assertTrue(ent.is_file())
self.assertFalse(ent.is_dir())
self.assertFalse(ent.is_symlink())
elif ent.name == 'test_dir':
self.assertFalse(ent.is_file())
self.assertTrue(ent.is_dir())
self.assertFalse(ent.is_symlink())
elif ent.name == 'link_test_file':
self.assertTrue(ent.is_file())
self.assertFalse(ent.is_file(follow_symlinks=False))
self.assertFalse(ent.is_dir())
self.assertFalse(ent.is_dir(follow_symlinks=False))
self.assertTrue(ent.is_symlink())
elif ent.name == 'link_test_dir':
self.assertFalse(ent.is_file())
self.assertFalse(ent.is_file(follow_symlinks=False))
self.assertTrue(ent.is_dir())
self.assertFalse(ent.is_dir(follow_symlinks=False))
self.assertTrue(ent.is_symlink())
elif ent.name == 'link_does_not_exist':
self.assertFalse(ent.is_file())
self.assertFalse(ent.is_file(follow_symlinks=False))
self.assertFalse(ent.is_dir())
self.assertFalse(ent.is_dir(follow_symlinks=False))
self.assertTrue(ent.is_symlink())
else:
self.fail('unexpected filename: ' + ent.name)
self.assertEqual(num_entries, 5)

1
vndk/tools/definition-tool/tests/testdata/test_scandir/link_does_not_exist vendored Symbolic link
View File

@@ -0,0 +1 @@
does_not_exist

1
vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_dir vendored Symbolic link
View File

@@ -0,0 +1 @@
test_dir

1
vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_file vendored Symbolic link
View File

@@ -0,0 +1 @@
test_file

0
vndk/tools/definition-tool/tests/testdata/test_scandir/test_dir/test_file vendored Normal file
View File

0
vndk/tools/definition-tool/tests/testdata/test_scandir/test_file vendored Normal file
View File

323
vndk/tools/definition-tool/vndk_definition_tool.py
View File

@@ -16,6 +16,7 @@ import re
import shutil
import stat
import struct
import subprocess
import sys
import zipfile
@@ -77,6 +78,98 @@ try:
except ImportError:
pass
try:
from tempfile import TemporaryDirectory
except ImportError:
import shutil
import tempfile
class TemporaryDirectory(object):
def __init__(self, suffix='', prefix='tmp', dir=None):
# pylint: disable=redefined-builtin
self.name = tempfile.mkdtemp(suffix, prefix, dir)
def __del__(self):
self.cleanup()
def __enter__(self):
return self.name
def __exit__(self, exc, value, tb):
self.cleanup()
def cleanup(self):
if self.name:
shutil.rmtree(self.name)
self.name = None
try:
from os import scandir
except ImportError:
import stat
import os
class DirEntry(object):
def __init__(self, name, path):
self.name = name
self.path = path
self._stat = None
self._lstat = None
@staticmethod
def _stat_impl(path, follow_symlinks):
return os.stat(path) if follow_symlinks else os.lstat(path)
def stat(self, follow_symlinks=True):
attr = '_stat' if follow_symlinks else '_lstat'
stat_res = getattr(self, attr)
if stat_res is None:
stat_res = self._stat_impl(self.path, follow_symlinks)
setattr(self, attr, stat_res)
return stat_res
def is_dir(self, follow_symlinks=True):
try:
return stat.S_ISDIR(self.stat(follow_symlinks).st_mode)
except EnvironmentError:
return False
def is_file(self, follow_symlinks=True):
try:
return stat.S_ISREG(self.stat(follow_symlinks).st_mode)
except EnvironmentError:
return False
def is_symlink(self):
return stat.S_ISLNK(self.stat(follow_symlinks=False).st_mode)
def scandir(path):
for name in os.listdir(path):
yield DirEntry(name, os.path.join(path, name))
#------------------------------------------------------------------------------
# Print Function
#------------------------------------------------------------------------------
def print_sb(*args, **kwargs):
"""A print function that supports both str and bytes."""
sep = kwargs.get('sep', ' ')
end = kwargs.get('end', '\n')
out_file = kwargs.get('file', sys.stdout)
for i, arg in enumerate(args):
if i > 0:
out_file.write(sep)
if isinstance(arg, str):
out_file.write(arg)
elif isinstance(arg, bytes):
out_file.flush()
out_file.buffer.write(arg)
out_file.flush()
else:
out_file.write(str(arg))
out_file.write(end)
#------------------------------------------------------------------------------
# Modified UTF-8 Encoder and Decoder
@@ -1413,22 +1506,6 @@ class LibProperties(object):
# ELF Linker
#------------------------------------------------------------------------------
def is_accessible(path):
try:
mode = os.stat(path).st_mode
return (mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) != 0
except FileNotFoundError:
return False
def scan_accessible_files(root):
for base, _, files in os.walk(root):
for filename in files:
path = os.path.join(base, filename)
if is_accessible(path):
yield path
def is_zipfile(path):
# zipfile.is_zipfile() tries to find the zip header in the file. But we
# only want to scan the zip file that starts with the magic word. Thus,
@@ -1452,24 +1529,134 @@ def scan_zip_file(zip_file_path):
zip_file.open(name, 'r').read())
def scan_elf_files(root, unzip_files=True):
"""Scan all ELF files under a directory."""
for path in scan_accessible_files(root):
# If this is a zip file and unzip_file is true, scan the ELF files in
# the zip file.
if unzip_files and is_zipfile(path):
for path, content in scan_zip_file(path):
try:
yield (path, ELF.loads(content))
except ELFError:
pass
continue
def dump_ext4_img(img_file_path, out_dir):
if ' ' in out_dir:
raise ValueError('out_dir must not have space character')
# Load ELF from the path.
try:
yield (path, ELF.load(path))
except ELFError:
pass
cmd = ['debugfs', img_file_path, '-R', 'rdump / ' + out_dir]
# Run the debugfs command.
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
raise subprocess.CalledProcessError(proc.returncode, cmd, stderr)
# Print error messages if they are not the ones that should be ignored.
for line in stderr.splitlines():
if line.startswith(b'debugfs '):
continue
if b'Operation not permitted while changing ownership of' in line:
continue
print_sb('error: debugfs:', line, file=sys.stderr)
def is_accessible(path):
try:
mode = os.stat(path).st_mode
return (mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) != 0
except FileNotFoundError:
return False
def scan_ext4_image(img_file_path, mount_point, unzip_files):
"""Scan all ELF files in the ext4 image."""
with TemporaryDirectory() as tmp_dir:
dump_ext4_img(img_file_path, tmp_dir)
for path, elf in scan_elf_files(tmp_dir, mount_point, unzip_files):
yield path, elf
def scan_apex_dir(apex_collection_root, apex_dir, unzip_files):
# Read the manifest file.
manifest_file_path = os.path.join(apex_dir, 'apex_manifest.json')
try:
with open(manifest_file_path, 'r') as manifest_file:
manifest = json.load(manifest_file)
except FileNotFoundError:
print('error: Failed to find apex manifest: {}'
.format(manifest_file_path), file=sys.stderr)
return
# Read the module name.
try:
apex_name = manifest['name']
except KeyError:
print('error: Failed to read apex name from manifest: {}'
.format(manifest_file_path), file=sys.stderr)
return
# Scan the payload (or the flatten payload).
mount_point = os.path.join(apex_collection_root, apex_name)
img_file_path = os.path.join(apex_dir, 'apex_payload.img')
if os.path.exists(img_file_path):
for path, elf in scan_ext4_image(img_file_path, mount_point,
unzip_files):
yield path, elf
else:
for path, elf in scan_elf_files(apex_dir, mount_point, unzip_files):
yield path, elf
def scan_apex_file(apex_collection_root, apex_zip_file, unzip_files):
with TemporaryDirectory() as tmp_dir:
with zipfile.ZipFile(apex_zip_file) as zip_file:
zip_file.extractall(tmp_dir)
for path, elf in scan_apex_dir(apex_collection_root, tmp_dir,
unzip_files):
yield path, elf
def scan_apex_files(apex_collection_root, unzip_files):
for ent in scandir(apex_collection_root):
if ent.is_dir():
for path, elf in scan_apex_dir(apex_collection_root, ent.path,
unzip_files):
yield path, elf
elif ent.is_file() and ent.name.endswith('.apex'):
for path, elf in scan_apex_file(apex_collection_root, ent.path,
unzip_files):
yield path, elf
def scan_elf_files(root, mount_point=None, unzip_files=True):
"""Scan all ELF files under a directory."""
if mount_point:
root_prefix_len = len(root) + 1
def norm_path(path):
return os.path.join(mount_point, path[root_prefix_len:])
else:
def norm_path(path):
return path
for base, dirnames, filenames in os.walk(root):
if base == root and 'apex' in dirnames:
dirnames.remove('apex')
for path, elf in scan_apex_files(os.path.join(root, 'apex'),
unzip_files):
yield (path, elf)
for filename in filenames:
path = os.path.join(base, filename)
if not is_accessible(path):
continue
# If this is a zip file and unzip_file is true, scan the ELF files
# in the zip file.
if unzip_files and is_zipfile(path):
for path, content in scan_zip_file(path):
try:
yield (norm_path(path), ELF.loads(content))
except ELFError:
pass
continue
# Load ELF from the path.
try:
yield (norm_path(path), ELF.load(path))
except ELFError:
pass
PT_SYSTEM = 0
@@ -1999,6 +2186,8 @@ class ELFLinker(object):
self.ro_vndk_version = ro_vndk_version
self.apex_module_names = set()
def _add_lib_to_lookup_dict(self, lib):
self.lib_pt[lib.partition].add(lib.path, lib)
@@ -2008,6 +2197,12 @@ class ELFLinker(object):
self.lib_pt[lib.partition].remove(lib)
def _rename_lib(self, lib, new_path):
self._remove_lib_from_lookup_dict(lib)
lib.path = new_path
self._add_lib_to_lookup_dict(lib)
def add_lib(self, partition, path, elf):
lib = ELFLinkData(partition, path, elf,
self.tagged_paths.get_path_tag_bit(path))
@@ -2102,7 +2297,7 @@ class ELFLinker(object):
ignored_patt = ELFLinker._compile_path_matcher(
root, ignored_subdirs)
for path, elf in scan_elf_files(root, unzip_files):
for path, elf in scan_elf_files(root, unzip_files=unzip_files):
# Ignore ELF files with unknown machine ID (eg. DSP).
if elf.e_machine not in ELF.ELF_MACHINES:
continue
@@ -2194,12 +2389,21 @@ class ELFLinker(object):
self._resolve_lib_deps(lib, resolver, generic_refs)
def _get_apex_bionic_search_paths(self, lib_dir):
return ['/apex/com.android.runtime/' + lib_dir + '/bionic']
def _get_apex_search_paths(self, lib_dir):
return ['/apex/' + name + '/' + lib_dir
for name in sorted(self.apex_module_names)]
def _get_system_search_paths(self, lib_dir):
return [
'/system/' + lib_dir,
# To find violating dependencies to vendor partitions.
'/vendor/' + lib_dir,
]
apex_lib_dirs = (self._get_apex_search_paths(lib_dir) +
self._get_apex_bionic_search_paths(lib_dir))
system_lib_dirs = ['/system/' + lib_dir, '/system/product/' + lib_dir]
vendor_lib_dirs = ['/vendor/' + lib_dir]
return apex_lib_dirs + system_lib_dirs + vendor_lib_dirs
def _get_vendor_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs):
@@ -2208,28 +2412,30 @@ class ELFLinker(object):
'/vendor/' + lib_dir + '/egl',
'/vendor/' + lib_dir,
]
system_lib_dirs = [
# For degenerated VNDK libs.
'/system/' + lib_dir,
]
return vendor_lib_dirs + vndk_sp_dirs + vndk_dirs + system_lib_dirs
# For degenerated VNDK libs.
apex_lib_dirs = (self._get_apex_search_paths(lib_dir) +
self._get_apex_bionic_search_paths(lib_dir))
system_lib_dirs = ['/system/' + lib_dir]
return (vendor_lib_dirs + vndk_sp_dirs + vndk_dirs + apex_lib_dirs +
system_lib_dirs)
def _get_vndk_sp_search_paths(self, lib_dir, vndk_sp_dirs):
# To find missing dependencies or LL-NDK.
fallback_lib_dirs = [
# To find missing VNDK-SP dependencies.
'/vendor/' + lib_dir,
# To find missing VNDK-SP dependencies or LL-NDK.
'/system/' + lib_dir,
]
fallback_lib_dirs += self._get_apex_search_paths(lib_dir)
fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir)
return vndk_sp_dirs + fallback_lib_dirs
def _get_vndk_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs):
fallback_lib_dirs = [
# To find missing VNDK dependencies or LL-NDK.
'/system/' + lib_dir,
]
# To find missing dependencies or LL-NDK.
fallback_lib_dirs = ['/system/' + lib_dir]
fallback_lib_dirs += self._get_apex_search_paths(lib_dir)
fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir)
return vndk_sp_dirs + vndk_dirs + fallback_lib_dirs
@@ -2353,6 +2559,22 @@ class ELFLinker(object):
lib.hide_dlopen_dep(dep)
def rewrite_apex_modules(self):
"""Rename ELF files under `/system/apex/${name}.apex` to
`/apex/${name}/...` and collect apex module names."""
APEX_PREFIX = '/system/apex/'
APEX_PREFIX_LEN = len(APEX_PREFIX)
for lib in list(self.all_libs()):
if not lib.path.startswith(APEX_PREFIX):
continue
apex_name_end = lib.path.find('/', APEX_PREFIX_LEN)
apex_name = lib.path[APEX_PREFIX_LEN:apex_name_end]
self.apex_module_names.add(apex_name)
self._rename_lib(lib, '/apex/' + lib.path[APEX_PREFIX_LEN:])
@staticmethod
def _parse_action_on_ineligible_lib(arg):
follow = False
@@ -2708,6 +2930,7 @@ class ELFLinker(object):
for path in extra_deps:
graph.add_dlopen_deps(path)
graph.rewrite_apex_modules()
graph.resolve_deps(generic_refs)
return graph