From d31eb32940d516771da392939f6e6aecb7465f69 Mon Sep 17 00:00:00 2001 From: Logan Chien Date: Wed, 31 May 2017 19:55:12 +0800 Subject: [PATCH 1/4] vndk-def: Introduce better eligible list This commit adds TaggedLibDict to read eligible list from CSV files and decide whether a lib is visible to another module. Note: The file format for eligible-list.csv has been changed. Prepend library names with `/system/${LIB}/`, e.g. `/system/${LIB}/libc.so`. Test: ./tests/test_tagged_lib_dict.py Test: ./tests/run.py Change-Id: Id415642b7a5609a3cf791364519a443a6c17c1a3 --- .../definition-tool/tests/test_tagged_dict.py | 479 ++++++++++++++++++ .../definition-tool/vndk_definition_tool.py | 285 +++++++---- 2 files changed, 664 insertions(+), 100 deletions(-) create mode 100755 vndk/tools/definition-tool/tests/test_tagged_dict.py diff --git a/vndk/tools/definition-tool/tests/test_tagged_dict.py b/vndk/tools/definition-tool/tests/test_tagged_dict.py new file mode 100755 index 000000000..92150e95e --- /dev/null +++ b/vndk/tools/definition-tool/tests/test_tagged_dict.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import tempfile +import unittest + +from compat import StringIO +from vndk_definition_tool import TaggedDict, TaggedPathDict, TaggedLibDict, \ + NUM_PARTITIONS, PT_SYSTEM, PT_VENDOR + + +_TEST_DATA = '''Path,Tag +/system/lib/lib_ll_ndk.so,ll-ndk +/system/lib/lib_ll_ndk_indirect.so,ll-ndk-indirect +/system/lib/lib_sp_ndk.so,sp-ndk +/system/lib/lib_sp_ndk_indirect.so,sp-ndk-indirect +/system/lib/lib_vndk_sp.so,vndk-sp +/system/lib/lib_vndk_sp_indirect.so,vndk-sp-indirect +/system/lib/lib_vndk_sp_indirect_private.so,vndk-sp-indirect-private +/system/lib/lib_vndk.so,vndk +/system/lib/lib_fwk_only.so,fwk-only +/system/lib/lib_fwk_only_rs.so,fwk-only-rs +/vendor/lib/lib_sp_hal.so,sp-hal +/vendor/lib/lib_sp_hal_dep.so,sp-hal-dep +/vendor/lib/lib_vnd_only.so,vnd-only +/system/lib/lib_remove.so,remove +/system/lib/lib_hl_ndk.so,hl-ndk +/system/lib/lib_vndk_indirect.so,vndk-indirect +/system/lib/lib_vndk_sp_both.so,vndk-sp-both +/system/lib/lib_vndk_sp_hal.so,vndk-sp-hal +''' + +_TEST_DATA_ALIAS_PATHS = { + '/system/lib/lib_hl_ndk.so', + '/system/lib/lib_vndk_sp_both.so', + '/system/lib/lib_vndk_sp_hal.so', + '/system/lib/lib_vndk_indirect.so' +} + + +class TaggedDictTest(unittest.TestCase): + def test_normalize_tag(self): + self.assertEqual('ll_ndk', TaggedDict._normalize_tag('ll-ndk')) + self.assertEqual('ll_ndk', TaggedDict._normalize_tag('LL-NDK')) + self.assertEqual('ll_ndk', TaggedDict._normalize_tag('LL_NDK')) + with self.assertRaises(ValueError): + TaggedDict._normalize_tag('unknown-lib-tag') + + + def _check_tag_visibility(self, d, from_tag, visible_tags): + for to_tag in TaggedDict.TAGS: + self.assertEqual(d.is_tag_visible(from_tag, to_tag), + to_tag in visible_tags) + + + def test_is_tag_visible(self): + d = TaggedDict() + + # LL-NDK + visible_tags = {'ll_ndk', 'll_ndk_indirect'} + self._check_tag_visibility(d, 'll_ndk', visible_tags) + self._check_tag_visibility(d, 'll_ndk_indirect', visible_tags) + + # SP-NDK + visible_tags = {'ll_ndk', 'll_ndk_indirect', + 'sp_ndk', 'sp_ndk_indirect'} + self._check_tag_visibility(d, 'sp_ndk', visible_tags) + self._check_tag_visibility(d, 'sp_ndk_indirect', visible_tags) + + # VNDK-SP + visible_tags = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', + 'vndk_sp_indirect_private', 'fwk_only_rs'} + self._check_tag_visibility(d, 'vndk_sp', visible_tags) + self._check_tag_visibility(d, 'vndk_sp_indirect', visible_tags) + self._check_tag_visibility(d, 'vndk_sp_indirect_private', visible_tags) + + # VNDK + visible_tags = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', + 'vndk'} + self._check_tag_visibility(d, 'vndk', visible_tags) + + # FWK-ONLY + visible_tags = {'ll_ndk', 'll_ndk_indirect', 'sp_ndk', + 'sp_ndk_indirect', 'vndk_sp', 'vndk_sp_indirect', + 'vndk_sp_indirect_private', 'vndk', 'fwk_only', + 'fwk_only_rs', 'sp_hal'} + self._check_tag_visibility(d, 'fwk_only', visible_tags) + self._check_tag_visibility(d, 'fwk_only_rs', visible_tags) + + # SP-HAL + visible_tags = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'sp_hal', 'sp_hal_dep'} + self._check_tag_visibility(d, 'sp_hal', visible_tags) + self._check_tag_visibility(d, 'sp_hal_dep', visible_tags) + + # VND-ONLY + visible_tags = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', + 'vndk', 'sp_hal', 'sp_hal_dep', 'vnd_only'} + self._check_tag_visibility(d, 'vnd_only', visible_tags) + + # Remove + self._check_tag_visibility(d, 'remove', set()) + + +class TaggedPathDictTest(unittest.TestCase): + def test_enumerate_paths(self): + self.assertEqual( + ['/system/lib/libc.so'], + list(TaggedPathDict._enumerate_paths('/system/lib/libc.so'))) + + self.assertEqual( + ['/system/lib/libc.so', '/system/lib64/libc.so'], + list(TaggedPathDict._enumerate_paths('/system/${LIB}/libc.so'))) + + + def test_load_from_csv_empty(self): + try: + TaggedPathDict().load_from_csv(StringIO('')) + except StopIteration: + self.fail('empty file should be considered as a valid input') + + + def test_load_from_csv_with_header(self): + fp = StringIO('Path,Tag\nliba.so,fwk-only\n') + d = TaggedPathDict() + d.load_from_csv(fp) + self.assertIn('liba.so', d.fwk_only) + + fp = StringIO('Tag,Path\nfwk-only,liba.so\n') + d = TaggedPathDict() + d.load_from_csv(fp) + self.assertIn('liba.so', d.fwk_only) + + + def test_load_from_csv_without_header(self): + fp = StringIO('liba.so,fwk-only\n') + d = TaggedPathDict() + d.load_from_csv(fp) + self.assertIn('liba.so', d.fwk_only) + + + def _check_test_data_loaded(self, d): + # Paths + self.assertIn('/system/lib/lib_ll_ndk.so', d.ll_ndk) + self.assertIn('/system/lib/lib_ll_ndk_indirect.so', d.ll_ndk_indirect) + self.assertIn('/system/lib/lib_sp_ndk.so', d.sp_ndk) + self.assertIn('/system/lib/lib_sp_ndk_indirect.so', d.sp_ndk_indirect) + self.assertIn('/system/lib/lib_vndk_sp.so', d.vndk_sp) + self.assertIn('/system/lib/lib_vndk_sp_indirect.so', d.vndk_sp_indirect) + self.assertIn('/system/lib/lib_vndk_sp_indirect_private.so', + d.vndk_sp_indirect_private) + self.assertIn('/system/lib/lib_vndk.so', d.vndk) + self.assertIn('/system/lib/lib_fwk_only.so', d.fwk_only) + self.assertIn('/system/lib/lib_fwk_only_rs.so', d.fwk_only_rs) + self.assertIn('/vendor/lib/lib_sp_hal.so', d.sp_hal) + self.assertIn('/vendor/lib/lib_sp_hal_dep.so', d.sp_hal_dep) + self.assertIn('/vendor/lib/lib_vnd_only.so', d.vnd_only) + self.assertIn('/system/lib/lib_remove.so', d.remove) + + # Aliases + self.assertIn('/system/lib/lib_hl_ndk.so', d.fwk_only) + self.assertIn('/system/lib/lib_vndk_sp_both.so', d.vndk_sp) + self.assertIn('/system/lib/lib_vndk_sp_hal.so', d.vndk_sp) + self.assertIn('/system/lib/lib_vndk_indirect.so', d.vndk) + + + def test_load_from_csv_tags(self): + fp = StringIO(_TEST_DATA) + d = TaggedPathDict() + d.load_from_csv(fp) + self._check_test_data_loaded(d) + + + def test_create_from_csv(self): + d = TaggedPathDict.create_from_csv(StringIO(_TEST_DATA)) + self._check_test_data_loaded(d) + + + def test_create_from_csv_path(self): + with tempfile.NamedTemporaryFile('w+') as f: + f.write(_TEST_DATA) + f.flush() + + d = TaggedPathDict.create_from_csv_path(f.name) + self._check_test_data_loaded(d) + + + def test_get_path_tag(self): + fp = StringIO(_TEST_DATA) + d = TaggedPathDict() + d.load_from_csv(fp) + + self.assertEqual('ll_ndk', d.get_path_tag('/system/lib/lib_ll_ndk.so')) + self.assertEqual('ll_ndk_indirect', + d.get_path_tag('/system/lib/lib_ll_ndk_indirect.so')) + self.assertEqual('sp_ndk', d.get_path_tag('/system/lib/lib_sp_ndk.so')) + self.assertEqual('sp_ndk_indirect', + d.get_path_tag('/system/lib/lib_sp_ndk_indirect.so')) + self.assertEqual('vndk_sp', + d.get_path_tag('/system/lib/lib_vndk_sp.so')) + self.assertEqual('vndk_sp_indirect', + d.get_path_tag('/system/lib/lib_vndk_sp_indirect.so')) + self.assertEqual( + 'vndk_sp_indirect_private', + d.get_path_tag('/system/lib/lib_vndk_sp_indirect_private.so')) + self.assertEqual('vndk', d.get_path_tag('/system/lib/lib_vndk.so')) + self.assertEqual('fwk_only', + d.get_path_tag('/system/lib/lib_fwk_only.so')) + self.assertEqual('fwk_only_rs', + d.get_path_tag('/system/lib/lib_fwk_only_rs.so')) + self.assertEqual('sp_hal', + d.get_path_tag('/vendor/lib/lib_sp_hal.so')) + self.assertEqual('sp_hal_dep', + d.get_path_tag('/vendor/lib/lib_sp_hal_dep.so')) + self.assertEqual('vnd_only', + d.get_path_tag('/vendor/lib/lib_vnd_only.so')) + self.assertEqual('remove', + d.get_path_tag('/system/lib/lib_remove.so')) + + # Aliases + self.assertEqual('fwk_only', + d.get_path_tag('/system/lib/lib_hl_ndk.so')) + self.assertEqual('vndk_sp', + d.get_path_tag('/system/lib/lib_vndk_sp_hal.so')) + self.assertEqual('vndk_sp', + d.get_path_tag('/system/lib/lib_vndk_sp_both.so')) + self.assertEqual('vndk', + d.get_path_tag('/system/lib/lib_vndk_indirect.so')) + + # Unmatched paths + self.assertEqual('fwk_only', d.get_path_tag('/system/lib/unknown.so')) + self.assertEqual('fwk_only', d.get_path_tag('/data/lib/unknown.so')) + self.assertEqual('vnd_only', d.get_path_tag('/vendor/lib/unknown.so')) + + + def _check_path_visibility(self, d, all_paths, from_paths, visible_paths): + for from_path in from_paths: + for to_path in all_paths: + self.assertEqual(d.is_path_visible(from_path, to_path), + to_path in visible_paths) + + + def test_is_path_visible(self): + fp = StringIO(_TEST_DATA) + d = TaggedPathDict() + d.load_from_csv(fp) + + # Collect path universe set. + all_paths = set() + for tag in TaggedPathDict.TAGS: + all_paths |= getattr(d, tag) + all_paths -= _TEST_DATA_ALIAS_PATHS + + # LL-NDK + from_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_ll_ndk_indirect.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_ll_ndk_indirect.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # SP-NDK + from_paths = { + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_sp_ndk_indirect.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_ll_ndk_indirect.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_sp_ndk_indirect.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # VNDK-SP + from_paths = { + '/system/lib/lib_vndk_sp.so', + '/system/lib/lib_vndk_sp_indirect.so', + '/system/lib/lib_vndk_sp_indirect_private.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_vndk_sp.so', + '/system/lib/lib_vndk_sp_indirect.so', + '/system/lib/lib_vndk_sp_indirect_private.so', + '/system/lib/lib_fwk_only_rs.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # VNDK + from_paths = { + '/system/lib/lib_vndk.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_vndk_sp.so', + '/system/lib/lib_vndk_sp_indirect.so', + '/system/lib/lib_vndk.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # FWK-ONLY + from_paths = { + '/system/lib/lib_fwk_only.so', + '/system/lib/lib_fwk_only_rs.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_ll_ndk_indirect.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_sp_ndk_indirect.so', + '/system/lib/lib_vndk_sp.so', + '/system/lib/lib_vndk_sp_indirect.so', + '/system/lib/lib_vndk_sp_indirect_private.so', + '/system/lib/lib_vndk.so', + '/system/lib/lib_fwk_only.so', + '/system/lib/lib_fwk_only_rs.so', + '/vendor/lib/lib_sp_hal.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # SP-HAL + from_paths = { + '/vendor/lib/lib_sp_hal.so', + '/vendor/lib/lib_sp_hal_dep.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_vndk_sp.so', + '/vendor/lib/lib_sp_hal.so', + '/vendor/lib/lib_sp_hal_dep.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + # VND-ONLY + from_paths = { + '/vendor/lib/lib_vnd_only.so', + } + visible_paths = { + '/system/lib/lib_ll_ndk.so', + '/system/lib/lib_sp_ndk.so', + '/system/lib/lib_vndk_sp.so', + '/system/lib/lib_vndk_sp_indirect.so', + '/system/lib/lib_vndk.so', + '/vendor/lib/lib_sp_hal.so', + '/vendor/lib/lib_sp_hal_dep.so', + '/vendor/lib/lib_vnd_only.so', + } + self._check_path_visibility(d, all_paths, from_paths, visible_paths) + + +class MockSPLibResult(object): + def __init__(self, sp_hal, sp_hal_dep): + self.sp_hal = sp_hal + self.sp_hal_dep = sp_hal_dep + + +class MockELFLinkData(object): + def __init__(self, path): + self.path = path + + +class MockELFGraph(object): + def __init__(self): + self.lib_pt = [dict() for i in range(NUM_PARTITIONS)] + + def add(self, path): + partition = PT_VENDOR if path.startswith('/vendor') else PT_SYSTEM + lib = MockELFLinkData(path) + self.lib_pt[partition][path] = lib + return lib + + def compute_sp_lib(self, generic_refs=None): + vendor_libs = self.lib_pt[PT_VENDOR].values() + return MockSPLibResult( + set(v for v in vendor_libs if 'lib_sp_hal.so' in v.path), + set(v for v in vendor_libs if 'lib_sp_hal_dep.so' in v.path)) + + +class TaggedLibDictTest(unittest.TestCase): + def setUp(self): + self.tagged_paths = TaggedPathDict.create_from_csv(StringIO(_TEST_DATA)) + + self.graph = MockELFGraph() + + self.lib_ll_ndk = self.graph.add('/system/lib/lib_ll_ndk.so') + self.lib_ll_ndk_indirect = \ + self.graph.add('/system/lib/lib_ll_ndk_indirect.so') + + self.lib_sp_ndk = self.graph.add('/system/lib/lib_sp_ndk.so') + self.lib_sp_ndk_indirect = \ + self.graph.add('/system/lib/lib_sp_ndk_indirect.so') + + self.lib_vndk_sp = self.graph.add('/system/lib/lib_vndk_sp.so') + self.lib_vndk_sp_indirect = \ + self.graph.add('/system/lib/lib_vndk_sp_indirect.so') + self.lib_vndk_sp_indirect_private = \ + self.graph.add('/system/lib/lib_vndk_sp_indirect_private.so') + + self.lib_vndk = self.graph.add('/system/lib/lib_vndk.so') + + self.lib_fwk_only = self.graph.add('/system/lib/lib_fwk_only.so') + self.lib_fwk_only_rs = self.graph.add('/system/lib/lib_fwk_only_rs.so') + + self.lib_sp_hal = self.graph.add('/vendor/lib/lib_sp_hal.so') + self.lib_sp_hal_dep = self.graph.add('/vendor/lib/lib_sp_hal_dep.so') + + self.lib_vnd_only = self.graph.add('/vendor/lib/lib_vnd_only.so') + + self.tagged_libs = \ + TaggedLibDict.create_from_graph(self.graph, self.tagged_paths) + + + def test_create_from_graph(self): + self.assertIn(self.lib_ll_ndk, self.tagged_libs.ll_ndk) + self.assertIn(self.lib_ll_ndk_indirect, + self.tagged_libs.ll_ndk_indirect) + self.assertIn(self.lib_sp_ndk, self.tagged_libs.sp_ndk) + self.assertIn(self.lib_sp_ndk_indirect, + self.tagged_libs.sp_ndk_indirect) + + self.assertIn(self.lib_vndk_sp, self.tagged_libs.vndk_sp) + self.assertIn(self.lib_vndk_sp_indirect, + self.tagged_libs.vndk_sp_indirect) + self.assertIn(self.lib_vndk_sp_indirect_private, + self.tagged_libs.vndk_sp_indirect_private) + + self.assertIn(self.lib_vndk, self.tagged_libs.vndk) + + self.assertIn(self.lib_fwk_only, self.tagged_libs.fwk_only) + self.assertIn(self.lib_fwk_only_rs, self.tagged_libs.fwk_only_rs) + + self.assertIn(self.lib_sp_hal, self.tagged_libs.sp_hal) + self.assertIn(self.lib_sp_hal_dep, self.tagged_libs.sp_hal_dep) + self.assertIn(self.lib_vnd_only, self.tagged_libs.vnd_only) + + + def test_get_path_tag(self): + d = self.tagged_libs + + self.assertEqual('ll_ndk', d.get_path_tag(self.lib_ll_ndk)) + self.assertEqual('ll_ndk_indirect', + d.get_path_tag(self.lib_ll_ndk_indirect)) + self.assertEqual('sp_ndk', d.get_path_tag(self.lib_sp_ndk)) + self.assertEqual('sp_ndk_indirect', + d.get_path_tag(self.lib_sp_ndk_indirect)) + self.assertEqual('vndk_sp', d.get_path_tag(self.lib_vndk_sp)) + self.assertEqual('vndk_sp_indirect', + d.get_path_tag(self.lib_vndk_sp_indirect)) + self.assertEqual('vndk_sp_indirect_private', + d.get_path_tag(self.lib_vndk_sp_indirect_private)) + self.assertEqual('vndk', d.get_path_tag(self.lib_vndk)) + self.assertEqual('fwk_only', d.get_path_tag(self.lib_fwk_only)) + self.assertEqual('fwk_only_rs', d.get_path_tag(self.lib_fwk_only_rs)) + self.assertEqual('sp_hal', d.get_path_tag(self.lib_sp_hal)) + self.assertEqual('sp_hal_dep', d.get_path_tag(self.lib_sp_hal_dep)) + self.assertEqual('vnd_only', d.get_path_tag(self.lib_vnd_only)) + + # Unmatched paths + tag = d.get_path_tag(MockELFLinkData('/system/lib/unknown.so')) + self.assertEqual('fwk_only', tag) + tag = d.get_path_tag(MockELFLinkData('/data/lib/unknown.so')) + self.assertEqual('fwk_only', tag) + tag = d.get_path_tag(MockELFLinkData('/vendor/lib/unknown.so')) + self.assertEqual('vnd_only', tag) + + +if __name__ == '__main__': + unittest.main() diff --git a/vndk/tools/definition-tool/vndk_definition_tool.py b/vndk/tools/definition-tool/vndk_definition_tool.py index 5a9684232..d4bd06bb3 100755 --- a/vndk/tools/definition-tool/vndk_definition_tool.py +++ b/vndk/tools/definition-tool/vndk_definition_tool.py @@ -2125,10 +2125,165 @@ class DepsClosureCommand(ELFGraphCommand): return 0 -TAGGED_LIB_DICT_FIELDS = ('ll_ndk', 'sp_ndk', 'hl_ndk', 'ndk_indirect', - 'vndk_sp', 'vndk', 'vndk_indirect', 'fwk_only') +class TaggedDict(object): + TAGS = { + 'll_ndk', 'll_ndk_indirect', 'sp_ndk', 'sp_ndk_indirect', + 'vndk_sp', 'vndk_sp_indirect', 'vndk_sp_indirect_private', + 'vndk', + 'fwk_only', 'fwk_only_rs', + 'sp_hal', 'sp_hal_dep', + 'vnd_only', + 'remove', + } + + _TAG_ALIASES = { + 'hl_ndk': 'fwk_only', # Treat HL-NDK as FWK-ONLY. + 'vndk_indirect': 'vndk', # Legacy + 'vndk_sp_hal': 'vndk_sp', # Legacy + 'vndk_sp_both': 'vndk_sp', # Legacy + } + + @classmethod + def _normalize_tag(cls, tag): + tag = tag.lower().replace('-', '_') + tag = cls._TAG_ALIASES.get(tag, tag) + if tag not in cls.TAGS: + raise ValueError('unknown lib tag ' + tag) + return tag + + _LL_NDK_VIS = {'ll_ndk', 'll_ndk_indirect'} + _SP_NDK_VIS = {'ll_ndk', 'll_ndk_indirect', 'sp_ndk', 'sp_ndk_indirect'} + _VNDK_SP_VIS = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', + 'vndk_sp_indirect_private', 'fwk_only_rs'} + _FWK_ONLY_VIS = {'ll_ndk', 'll_ndk_indirect', 'sp_ndk', 'sp_ndk_indirect', + 'vndk_sp', 'vndk_sp_indirect', 'vndk_sp_indirect_private', + 'vndk', 'fwk_only', 'fwk_only_rs', 'sp_hal'} + _SP_HAL_VIS = {'ll_ndk', 'sp_ndk', 'vndk_sp', 'sp_hal', 'sp_hal_dep'} + + _TAG_VISIBILITY = { + 'll_ndk': _LL_NDK_VIS, + 'll_ndk_indirect': _LL_NDK_VIS, + 'sp_ndk': _SP_NDK_VIS, + 'sp_ndk_indirect': _SP_NDK_VIS, + + 'vndk_sp': _VNDK_SP_VIS, + 'vndk_sp_indirect': _VNDK_SP_VIS, + 'vndk_sp_indirect_private': _VNDK_SP_VIS, + + 'vndk': {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', 'vndk'}, + + 'fwk_only': _FWK_ONLY_VIS, + 'fwk_only_rs': _FWK_ONLY_VIS, + + 'sp_hal': _SP_HAL_VIS, + 'sp_hal_dep': _SP_HAL_VIS, + + 'vnd_only': {'ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk_sp_indirect', + 'vndk', 'sp_hal', 'sp_hal_dep', 'vnd_only'}, + + 'remove': set(), + } + + del _LL_NDK_VIS, _SP_NDK_VIS, _VNDK_SP_VIS, _FWK_ONLY_VIS, _SP_HAL_VIS + + @classmethod + def is_tag_visible(cls, from_tag, to_tag): + return to_tag in cls._TAG_VISIBILITY[from_tag] + + def __init__(self): + self._path_tag = dict() + for tag in self.TAGS: + setattr(self, tag, set()) + + def add(self, tag, lib): + lib_set = getattr(self, tag) + lib_set.add(lib) + self._path_tag[lib] = tag + + def get_path_tag(self, lib): + try: + return self._path_tag[lib] + except KeyError: + return self.get_path_tag_default(lib) + + def get_path_tag_default(self, lib): + raise NotImplementedError() + + def is_path_visible(self, from_lib, to_lib): + return self.is_tag_visible(self.get_path_tag(from_lib), + self.get_path_tag(to_lib)) + + +class TaggedPathDict(TaggedDict): + def load_from_csv(self, fp): + reader = csv.reader(fp) + + # Read first row and check the existence of the header. + try: + row = next(reader) + except StopIteration: + return + + try: + path_col = row.index('Path') + tag_col = row.index('Tag') + except ValueError: + path_col = 0 + tag_col = 1 + self.add(self._normalize_tag(row[tag_col]), row[path_col]) + + # Read the rest of rows. + for row in reader: + self.add(self._normalize_tag(row[tag_col]), row[path_col]) + + @staticmethod + def create_from_csv(fp): + d = TaggedPathDict() + d.load_from_csv(fp) + return d + + @staticmethod + def create_from_csv_path(path): + with open(path, 'r') as fp: + return TaggedPathDict.create_from_csv(fp) + + @staticmethod + def _enumerate_paths(pattern): + if '${LIB}' in pattern: + yield pattern.replace('${LIB}', 'lib') + yield pattern.replace('${LIB}', 'lib64') + else: + yield pattern + + def add(self, tag, path): + for path in self._enumerate_paths(path): + super(TaggedPathDict, self).add(tag, path) + + def get_path_tag_default(self, path): + return 'vnd_only' if path.startswith('/vendor') else 'fwk_only' + + +class TaggedLibDict(TaggedDict): + @staticmethod + def create_from_graph(graph, tagged_paths, generic_refs=None): + d = TaggedLibDict() + + for lib in graph.lib_pt[PT_SYSTEM].values(): + d.add(tagged_paths.get_path_tag(lib.path), lib) + + sp_lib = graph.compute_sp_lib(generic_refs) + for lib in graph.lib_pt[PT_VENDOR].values(): + if lib in sp_lib.sp_hal: + d.add('sp_hal', lib) + elif lib in sp_lib.sp_hal_dep: + d.add('sp_hal_dep', lib) + else: + d.add('vnd_only', lib) + return d + + def get_path_tag_default(self, lib): + return 'vnd_only' if lib.path.startswith('/vendor') else 'fwk_only' -TaggedLibDict = collections.namedtuple('TaggedLibDict', TAGGED_LIB_DICT_FIELDS) class ModuleInfo(object): def __init__(self, module_info_path=None): @@ -2145,6 +2300,7 @@ class ModuleInfo(object): return module['path'] return [] + class CheckDepCommand(ELFGraphCommand): def __init__(self): super(CheckDepCommand, self).__init__( @@ -2157,42 +2313,6 @@ class CheckDepCommand(ELFGraphCommand): parser.add_argument('--module-info') - def _load_tag_file(self, tag_file_path): - res = TaggedLibDict(set(), set(), set(), set(), set(), set(), set(), - set()) - - mapping = { - 'll-ndk': res.ll_ndk, - 'll-ndk-indirect': res.ndk_indirect, - 'sp-ndk': res.sp_ndk, - 'sp-ndk-indirect': res.ndk_indirect, - 'hl-ndk': res.hl_ndk, - 'vndk-sp-hal': res.vndk_sp, - 'vndk-sp-both': res.vndk_sp, - 'vndk-sp-indirect': res.vndk, # Visible to non-SP-HAL - 'vndk': res.vndk, - 'vndk-indirect': res.vndk_indirect, - 'fwk-only': res.fwk_only, - 'remove': res.fwk_only, - } - - with open(tag_file_path, 'r') as tag_file: - csv_reader = csv.reader(tag_file) - for lib_name, tag in csv_reader: - mapping[tag.lower()].add(lib_name) - - return res - - def _get_tagged_libs(self, graph, tags): - res = TaggedLibDict(set(), set(), set(), set(), set(), set(), set(), - set()) - for lib in graph.lib_pt[PT_SYSTEM].values(): - lib_name = os.path.basename(lib.path) - for i, lib_set in enumerate(tags): - if lib_name in lib_set: - res[i].add(lib) - return res - @staticmethod def _dump_dep(lib, bad_deps, module_info): print(lib.path) @@ -2207,11 +2327,14 @@ class CheckDepCommand(ELFGraphCommand): """Check whether eligible sets are self-contained.""" num_errors = 0 - indirect_libs = tagged_libs.ndk_indirect + indirect_libs = (tagged_libs.ll_ndk_indirect | \ + tagged_libs.sp_ndk_indirect | \ + tagged_libs.vndk_sp_indirect_private | \ + tagged_libs.fwk_only_rs) eligible_libs = (tagged_libs.ll_ndk | tagged_libs.sp_ndk | \ - tagged_libs.vndk_sp | \ - tagged_libs.vndk | tagged_libs.vndk_indirect) + tagged_libs.vndk_sp | tagged_libs.vndk_sp_indirect | \ + tagged_libs.vndk) # Check eligible vndk is self-contained. for lib in sorted(eligible_libs): @@ -2247,8 +2370,8 @@ class CheckDepCommand(ELFGraphCommand): vendor_libs = graph.lib_pt[PT_VENDOR].values() eligible_libs = (tagged_libs.ll_ndk | tagged_libs.sp_ndk | \ - tagged_libs.vndk_sp | \ - tagged_libs.vndk | tagged_libs.vndk_indirect) + tagged_libs.vndk_sp | tagged_libs.vndk_sp_indirect | \ + tagged_libs.vndk) for lib in sorted(vendor_libs): bad_deps = [] @@ -2269,8 +2392,8 @@ class CheckDepCommand(ELFGraphCommand): args.vendor, args.vendor_dir_as_system, args.load_extra_deps) - tags = self._load_tag_file(args.tag_file) - tagged_libs = self._get_tagged_libs(graph, tags) + tagged_paths = TaggedPathDict.create_from_csv_path(args.tag_file) + tagged_libs = TaggedLibDict.create_from_graph(graph, tagged_paths) module_info = ModuleInfo(args.module_info) @@ -2290,57 +2413,19 @@ class DepGraphCommand(ELFGraphCommand): super(DepGraphCommand, self).add_argparser_options(parser) parser.add_argument('--tag-file', required=True) - parser.add_argument( - '--output', '-o', help='output directory') + parser.add_argument('--output', '-o', help='output directory') - def _load_tag_file(self, tag_file_path): - res = TaggedLibDict(set(), set(), set(), set(), set(), set(), set(), - set()) - - mapping = { - 'll-ndk': res.ll_ndk, - 'll-ndk-indirect': res.ndk_indirect, - 'sp-ndk': res.sp_ndk, - 'sp-ndk-indirect': res.ndk_indirect, - 'hl-ndk': res.hl_ndk, - 'vndk-sp-hal': res.vndk_sp, - 'vndk-sp-both': res.vndk_sp, - 'vndk-sp-indirect': res.vndk, # Visible to non-SP-HAL - 'vndk': res.vndk, - 'vndk-indirect': res.vndk_indirect, - 'fwk-only': res.fwk_only, - 'remove': set(), # Drop from system lib. Tag as vendor lib. - } - - with open(tag_file_path, 'r') as tag_file: - csv_reader = csv.reader(tag_file) - for lib_name, tag in csv_reader: - mapping[tag.lower()].add(os.path.basename(lib_name)) - - return res - - def _get_tag_from_lib(self, lib, tags): - # Tempororily add the TAG_DEP_TABLE to show permission - TAG_DEP_TABLE = { - 'vendor': ('ll_ndk', 'sp_ndk', 'vndk_sp', 'vndk', 'vndk_indirect'), - } - - tag_hierarchy = [] - eligible_for_vendor = TAG_DEP_TABLE['vendor'] - for tag in TAGGED_LIB_DICT_FIELDS: - if tag == 'vendor': - tag_hierarchy.append('vendor.private.bin') + def _get_tag_from_lib(self, lib, tagged_paths): + tag_hierarchy = dict() + for tag in TaggedPathDict.TAGS: + if tag in {'sp_hal', 'sp_hal_dep', 'vnd_only'}: + tag_hierarchy[tag] = 'vendor.private.{}'.format(tag) else: - pub = 'public' if tag in eligible_for_vendor else 'private' - tag_hierarchy.append('system.{}.{}'.format(pub, tag)) - lib_name = os.path.basename(lib.path) - if lib.partition == PT_SYSTEM: - for i, lib_set in enumerate(tags): - if lib_name in lib_set: - return tag_hierarchy[i] - return 'system.private.bin' - else: - return 'vendor.private.bin' + vendor_visible = TaggedPathDict.is_tag_visible('vnd_only', tag) + pub = 'public' if vendor_visible else 'private' + tag_hierarchy[tag] = 'system.{}.{}'.format(pub, tag) + + return tag_hierarchy[tagged_paths.get_path_tag(lib.path)] def _check_if_allowed(self, my_tag, other_tag): my = my_tag.split('.') @@ -2352,13 +2437,13 @@ class DepGraphCommand(ELFGraphCommand): return False return True - def _get_dep_graph(self, graph, tags): + def _get_dep_graph(self, graph, tagged_paths): data = [] violate_libs = dict() system_libs = graph.lib_pt[PT_SYSTEM].values() vendor_libs = graph.lib_pt[PT_VENDOR].values() for lib in itertools.chain(system_libs, vendor_libs): - tag = self._get_tag_from_lib(lib, tags) + tag = self._get_tag_from_lib(lib, tagged_paths) violate_count = 0 lib_item = { 'name': lib.path, @@ -2368,7 +2453,7 @@ class DepGraphCommand(ELFGraphCommand): } for dep in lib.deps: if self._check_if_allowed(tag, - self._get_tag_from_lib(dep, tags)): + self._get_tag_from_lib(dep, tagged_paths)): lib_item['depends'].append(dep.path) else: lib_item['violates'].append(dep.path) @@ -2386,8 +2471,8 @@ class DepGraphCommand(ELFGraphCommand): args.vendor, args.vendor_dir_as_system, args.load_extra_deps) - tags = self._load_tag_file(args.tag_file) - data, violate_libs = self._get_dep_graph(graph, tags) + tagged_paths = TaggedPathDict.create_from_csv_path(args.tag_file) + data, violate_libs = self._get_dep_graph(graph, tagged_paths) data.sort(key=lambda lib_item: (lib_item['tag'], lib_item['violate_count'])) for libs in violate_libs.values(): From 7e723126e4d94e03d4a15de98d5aaceca682cc5e Mon Sep 17 00:00:00 2001 From: Logan Chien Date: Thu, 25 May 2017 01:01:53 +0800 Subject: [PATCH 2/4] vndk-def: Implement degenerated VNDK for o-release This commit implements the degenerated directory layout for o-release. This commit also simplifies the output significantly. Test: Run vndk_definition_tool.py against o-release images. Change-Id: I5aad29fa4ff7e819778d26d2e2011af1be61bb58 --- .../datasets/minimum_dlopen_deps.txt | 12 + .../definition-tool/tests/test_elf_linker.py | 90 +- .../tests/test_generic_refs.py | 6 + vndk/tools/definition-tool/tests/test_vndk.py | 197 ---- .../definition-tool/vndk_definition_tool.py | 869 ++++++++---------- 5 files changed, 435 insertions(+), 739 deletions(-) create mode 100644 vndk/tools/definition-tool/datasets/minimum_dlopen_deps.txt delete mode 100755 vndk/tools/definition-tool/tests/test_vndk.py diff --git a/vndk/tools/definition-tool/datasets/minimum_dlopen_deps.txt b/vndk/tools/definition-tool/datasets/minimum_dlopen_deps.txt new file mode 100644 index 000000000..b1f057cae --- /dev/null +++ b/vndk/tools/definition-tool/datasets/minimum_dlopen_deps.txt @@ -0,0 +1,12 @@ +/system/lib/libEGL.so:/system/lib/libEGL.so +/system/lib/libEGL.so:/system/lib/libGLESv1_CM.so +/system/lib/libEGL.so:/system/lib/libGLESv2.so +/system/lib/libc.so:/system/lib/libc_malloc_debug.so +/system/lib/libc.so:/system/lib/libicuuc.so +/system/lib/libc.so:/system/lib/libnetd_client.so +/system/lib64/libEGL.so:/system/lib64/libEGL.so +/system/lib64/libEGL.so:/system/lib64/libGLESv1_CM.so +/system/lib64/libEGL.so:/system/lib64/libGLESv2.so +/system/lib64/libc.so:/system/lib64/libc_malloc_debug.so +/system/lib64/libc.so:/system/lib64/libicuuc.so +/system/lib64/libc.so:/system/lib64/libnetd_client.so diff --git a/vndk/tools/definition-tool/tests/test_elf_linker.py b/vndk/tools/definition-tool/tests/test_elf_linker.py index 402646cec..68109f26f 100755 --- a/vndk/tools/definition-tool/tests/test_elf_linker.py +++ b/vndk/tools/definition-tool/tests/test_elf_linker.py @@ -132,17 +132,13 @@ class ELFLinkerTest(unittest.TestCase): self.assertEqual(['/system/lib64/libdl.so'], self._get_paths_from_nodes(nodes)) - def test_elf_class(self): + def test_elf_class_and_partitions(self): gb = self._create_normal_graph() graph = gb.graph - self.assertEqual(6, len(graph.lib32)) - self.assertEqual(6, len(graph.lib64)) - - def test_partitions(self): - gb = self._create_normal_graph() - graph = gb.graph - self.assertEqual(10, len(gb.graph.lib_pt[PT_SYSTEM])) - self.assertEqual(2, len(gb.graph.lib_pt[PT_VENDOR])) + self.assertEqual(5, len(graph.lib_pt[PT_SYSTEM].lib32)) + self.assertEqual(5, len(graph.lib_pt[PT_SYSTEM].lib64)) + self.assertEqual(1, len(graph.lib_pt[PT_VENDOR].lib32)) + self.assertEqual(1, len(graph.lib_pt[PT_VENDOR].lib64)) def test_deps(self): gb = self._create_normal_graph() @@ -169,7 +165,7 @@ class ELFLinkerTest(unittest.TestCase): graph = gb.graph # Check the unresolved symbols. - for lib_set in (graph.lib32, graph.lib64): + for lib_set in graph.lib_pt: for lib in lib_set.values(): self.assertEqual(set(), lib.unresolved_symbols) @@ -237,6 +233,27 @@ class ELFLinkerTest(unittest.TestCase): node = graph.get_lib('/vendor/lib64/libEGL.so') self.assertEqual([], self._get_paths_from_nodes(node.users)) + def test_compute_predefined_fwk_only_rs(self): + lib_names = ( + 'libft2', + 'libmediandk', + ) + + # Add VNDK-SP libraries. + gb = GraphBuilder() + for name in lib_names: + gb.add_multilib(PT_SYSTEM, name) + gb.resolve() + + # Compute FWK-ONLY-RS and check the result. + fwk_only_rs = gb.graph.compute_predefined_fwk_only_rs() + fwk_only_rs = set(lib.path for lib in fwk_only_rs) + + for lib_dir_name in ('lib', 'lib64'): + lib_dir = '/system/' + lib_dir_name + for name in lib_names: + self.assertIn(os.path.join(lib_dir, name + '.so'), fwk_only_rs) + def test_compute_predefined_vndk_sp(self): lib_names = ( 'android.hardware.graphics.allocator@2.0', @@ -288,7 +305,6 @@ class ELFLinkerTest(unittest.TestCase): lib_names = ( 'libbacktrace', 'libblas', - 'libft2', 'liblzma', 'libpng', 'libunwind', @@ -505,58 +521,6 @@ class ELFLinkerTest(unittest.TestCase): self.assertNotIn(libc_path, sp_ndk_indirect) - def test_find_existing_vndk(self): - gb = GraphBuilder() - - libpng32_core, libpng64_core = \ - gb.add_multilib(PT_SYSTEM, 'libpng', extra_dir='vndk-26') - libpng32_fwk, libpng64_fwk = \ - gb.add_multilib(PT_SYSTEM, 'libpng', extra_dir='vndk-26-ext') - - libjpeg32_core, libjpeg64_core = \ - gb.add_multilib(PT_SYSTEM, 'libjpeg', extra_dir='vndk-26') - libjpeg32_vnd, libjpeg64_vnd = \ - gb.add_multilib(PT_VENDOR, 'libjpeg', extra_dir='vndk-26-ext') - - gb.resolve() - - vndk_core, vndk_fwk_ext, vndk_vnd_ext = gb.graph.find_existing_vndk() - - expected_vndk_core = { - libpng32_core, libpng64_core, libjpeg32_core, libjpeg64_core} - expected_vndk_fwk_ext = {libpng32_fwk, libpng64_fwk} - expected_vndk_vnd_ext = {libjpeg32_vnd, libjpeg64_vnd} - - self.assertSetEqual(expected_vndk_core, vndk_core) - self.assertSetEqual(expected_vndk_fwk_ext, vndk_fwk_ext) - self.assertSetEqual(expected_vndk_vnd_ext, vndk_vnd_ext) - - def test_find_existing_vndk_without_version(self): - gb = GraphBuilder() - - libpng32_core, libpng64_core = \ - gb.add_multilib(PT_SYSTEM, 'libpng', extra_dir='vndk') - libpng32_fwk, libpng64_fwk = \ - gb.add_multilib(PT_SYSTEM, 'libpng', extra_dir='vndk-ext') - - libjpeg32_core, libjpeg64_core = \ - gb.add_multilib(PT_SYSTEM, 'libjpeg', extra_dir='vndk') - libjpeg32_vnd, libjpeg64_vnd = \ - gb.add_multilib(PT_VENDOR, 'libjpeg', extra_dir='vndk-ext') - - gb.resolve() - - vndk_core, vndk_fwk_ext, vndk_vnd_ext = gb.graph.find_existing_vndk() - - expected_vndk_core = { - libpng32_core, libpng64_core, libjpeg32_core, libjpeg64_core} - expected_vndk_fwk_ext = {libpng32_fwk, libpng64_fwk} - expected_vndk_vnd_ext = {libjpeg32_vnd, libjpeg64_vnd} - - self.assertSetEqual(expected_vndk_core, vndk_core) - self.assertSetEqual(expected_vndk_fwk_ext, vndk_fwk_ext) - self.assertSetEqual(expected_vndk_vnd_ext, vndk_vnd_ext) - def test_compute_vndk_cap(self): gb = GraphBuilder() diff --git a/vndk/tools/definition-tool/tests/test_generic_refs.py b/vndk/tools/definition-tool/tests/test_generic_refs.py index eb20c9675..c887d19d6 100755 --- a/vndk/tools/definition-tool/tests/test_generic_refs.py +++ b/vndk/tools/definition-tool/tests/test_generic_refs.py @@ -87,6 +87,12 @@ class GenericRefsTest(unittest.TestCase): self.assertTrue(self.ref.is_equivalent_lib(libc_eq)) + def test_has_same_name_lib(self): + self.assertTrue(self.ref.has_same_name_lib( + MockLib('/vendor/lib/libc.so', {}))) + self.assertFalse(self.ref.has_same_name_lib( + MockLib('/vendor/lib/lib_does_not_exist.so', {}))) + if __name__ == '__main__': unittest.main() diff --git a/vndk/tools/definition-tool/tests/test_vndk.py b/vndk/tools/definition-tool/tests/test_vndk.py deleted file mode 100755 index 773ac1245..000000000 --- a/vndk/tools/definition-tool/tests/test_vndk.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import print_function - -import os -import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import unittest - -from compat import StringIO -from vndk_definition_tool import (ELF, ELFLinker, PT_SYSTEM, PT_VENDOR, - GenericRefs, SPLibResult) - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -TESTDATA_DIR = os.path.join(SCRIPT_DIR ,'testdata', 'test_vndk') - -class MockBannedLibs(object): - def is_banned(self, name): - return False - -class ELFLinkerVNDKTest(unittest.TestCase): - def _get_paths_from_nodes(self, nodes): - return sorted([node.path for node in nodes]) - - def _create_graph_gr(self, input_dir, generic_refs_dir): - if not generic_refs_dir: - generic_refs = None - else: - generic_refs_dir = os.path.join(TESTDATA_DIR, generic_refs_dir) - generic_refs = GenericRefs.create_from_sym_dir(generic_refs_dir) - - input_dir = os.path.join(TESTDATA_DIR, input_dir) - - graph = ELFLinker.create_from_dump( - system_dirs=[os.path.join(input_dir, 'system')], - vendor_dirs=[os.path.join(input_dir, 'vendor')], - generic_refs=generic_refs) - - return (graph, generic_refs) - - def _create_graph_vndk(self, input_dir, generic_refs_dir): - graph, generic_refs = self._create_graph_gr(input_dir, generic_refs_dir) - - vndk = graph._compute_vndk( - sp_lib=SPLibResult(set(), set(), set(), set(), set(), set()), - vndk_customized_for_system=set(), - vndk_customized_for_vendor=set(), - generic_refs=generic_refs, - banned_libs=MockBannedLibs()) - - return (graph, vndk) - - def test_compute_vndk(self): - graph, vndk = self._create_graph_vndk('pre_treble', None) - - self.assertEqual(['/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_vnd_ext)) - - def test_compute_vndk_indirect_no_gr(self): - graph, vndk = self._create_graph_vndk('vndk_indirect', None) - - self.assertEqual(['/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - self.assertEqual(['/system/lib/vndk/libcutils_dep.so', - '/system/lib64/vndk/libcutils_dep.so'], - self._get_paths_from_nodes(vndk.vndk_indirect)) - - def test_compute_vndk_fwk_ext(self): - graph, vndk = self._create_graph_vndk('vndk_fwk_ext', 'vndk_gr') - - self.assertEqual(['/system/lib/vndk/libRS.so', - '/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libRS.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - self.assertEqual(['/system/lib/vndk-ext/libRS.so', - '/system/lib64/vndk-ext/libRS.so'], - self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_vnd_ext)) - - def test_compute_vndk_vnd_ext(self): - graph, vndk = self._create_graph_vndk('vndk_vnd_ext', 'vndk_gr') - - self.assertEqual(['/system/lib/vndk/libRS.so', - '/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libRS.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - self.assertEqual(['/vendor/lib/vndk-ext/libRS.so', - '/vendor/lib64/vndk-ext/libRS.so'], - self._get_paths_from_nodes(vndk.vndk_vnd_ext)) - - def test_compute_vndk_inward_customization(self): - graph, generic_refs = self._create_graph_gr( - 'vndk_inward_customization', 'vndk_gr') - - # Make sure libjpeg.so was loaded from the input dir. - libjpeg_32 = graph.get_lib('/system/lib/libjpeg.so') - self.assertIsNotNone(libjpeg_32) - libjpeg_64 = graph.get_lib('/system/lib64/libjpeg.so') - self.assertIsNotNone(libjpeg_64) - - # Compute vndk sets and move libraries to the correct directories. - vndk = graph._compute_vndk( - sp_lib=SPLibResult(set(), set(), set(), set(), set(), set()), - vndk_customized_for_system=set(), - vndk_customized_for_vendor=set(), - generic_refs=generic_refs, - banned_libs=MockBannedLibs()) - - # Check vndk-core libraries. - self.assertEqual(['/system/lib/vndk/libRS.so', - '/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libRS.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - - # Check vndk-indirect libraries. - self.assertEqual(['/system/lib/vndk/libjpeg.so', - '/system/lib64/vndk/libjpeg.so'], - self._get_paths_from_nodes(vndk.vndk_indirect)) - - # Check libjpeg.so (inward-customization) has been renamed. - self.assertIsNone(graph.get_lib('/system/lib/libjpeg.so')) - self.assertIsNone(graph.get_lib('/system/lib64/libjpeg.so')) - self.assertIs(libjpeg_32, - graph.get_lib('/system/lib/vndk/libjpeg.so')) - self.assertIs(libjpeg_64, - graph.get_lib('/system/lib64/vndk/libjpeg.so')) - - # Check the absence of vndk-ext libraries. - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - self.assertEqual([], self._get_paths_from_nodes(vndk.vndk_vnd_ext)) - - def test_compute_vndk_indirect_ext(self): - # This test case reveals a corner case that will break vndk-indirect - # computation. To reproduce the case, the following condition must be - # satisfied: - # - # 1. libA depends on libB. - # 2. libA is a vndk-fwk-ext. - # 3. libB is an outward-customized vndk which depends on non-AOSP libC. - # - # Both AOSP libA and libB will be added to vndk-core. But, - # unfortunately, libA will be resolved to libB in vndk-fwk-ext and this - # will break the vndk-indirect computation because libC is not in - # generic references. - - graph, vndk = self._create_graph_vndk('vndk_indirect_ext', - 'vndk_indirect_ext_gr') - - self.assertEqual(['/system/lib/vndk/libRS.so', - '/system/lib/vndk/libcutils.so', - '/system/lib64/vndk/libRS.so', - '/system/lib64/vndk/libcutils.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - - self.assertEqual(['/system/lib/vndk/libRS_internal.so', - '/system/lib64/vndk/libRS_internal.so'], - self._get_paths_from_nodes(vndk.vndk_indirect)) - - self.assertEqual(['/system/lib/vndk-ext/libRS_internal.so', - '/system/lib64/vndk-ext/libRS_internal.so'], - self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - - def test_compute_vndk_ext_deps(self): - # This test case reveals a bug in the vndk-core dependencies assertion. - # This will happen when libA and libB are added to both vndk-fwk-ext and - # vndk-vnd-ext in the first round, libA depends libC, libC depends - # libB. - - graph, vndk = self._create_graph_vndk('vndk_ext_dep', 'vndk_ext_dep_gr') - - self.assertEqual(['/system/lib/vndk/libA.so', - '/system/lib/vndk/libB.so', - '/system/lib/vndk/libC.so'], - self._get_paths_from_nodes(vndk.vndk_core)) - - self.assertEqual(['/system/lib/vndk-ext/libA.so', - '/system/lib/vndk-ext/libB.so', - '/system/lib/vndk-ext/libC.so'], - self._get_paths_from_nodes(vndk.vndk_fwk_ext)) - - self.assertEqual(['/vendor/lib/vndk-ext/libA.so', - '/vendor/lib/vndk-ext/libB.so', - '/vendor/lib/vndk-ext/libC.so'], - self._get_paths_from_nodes(vndk.vndk_vnd_ext)) - - -if __name__ == '__main__': - unittest.main() diff --git a/vndk/tools/definition-tool/vndk_definition_tool.py b/vndk/tools/definition-tool/vndk_definition_tool.py index d4bd06bb3..46c587f0b 100755 --- a/vndk/tools/definition-tool/vndk_definition_tool.py +++ b/vndk/tools/definition-tool/vndk_definition_tool.py @@ -4,6 +4,7 @@ from __future__ import print_function import argparse import collections +import copy import csv import itertools import json @@ -51,6 +52,58 @@ except ImportError: pass +#------------------------------------------------------------------------------ +# Collections +#------------------------------------------------------------------------------ + +def defaultnamedtuple(typename, field_names, default): + """Create a namedtuple type with default values. + + This function creates a namedtuple type which will fill in default value + when actual arguments to the constructor were omitted. + + >>> Point = defaultnamedtuple('Point', ['x', 'y'], 0) + >>> Point() + Point(x=0, y=0) + >>> Point(1) + Point(x=1, y=0) + >>> Point(1, 2) + Point(x=1, y=2) + >>> Point(x=1, y=2) + Point(x=1, y=2) + >>> Point(y=2, x=1) + Point(x=1, y=2) + + >>> PermSet = defaultnamedtuple('PermSet', 'allowed disallowed', set()) + >>> s = PermSet() + >>> s + PermSet(allowed=set(), disallowed=set()) + >>> s.allowed is not s.disallowed + True + >>> PermSet({1}) + PermSet(allowed={1}, disallowed=set()) + >>> PermSet({1}, {2}) + PermSet(allowed={1}, disallowed={2}) + """ + + if isinstance(field_names, str): + field_names = field_names.replace(',', ' ').split() + field_names = list(map(str, field_names)) + num_fields = len(field_names) + + base_cls = collections.namedtuple(typename, field_names) + def __new__(cls, *args, **kwargs): + args = list(args) + for i in range(len(args), num_fields): + arg = kwargs.get(field_names[i]) + if arg: + args.append(arg) + else: + args.append(copy.copy(default)) + return base_cls.__new__(cls, *args) + return type(typename, (base_cls,), {'__new__': __new__}) + + #------------------------------------------------------------------------------ # ELF Parser #------------------------------------------------------------------------------ @@ -621,31 +674,6 @@ PT_VENDOR = 1 NUM_PARTITIONS = 2 -VNDKResult = collections.namedtuple( - 'VNDKResult', - 'sp_hal sp_hal_dep vndk_sp_hal sp_ndk sp_ndk_indirect ' - 'vndk_sp_both ' - 'extra_vendor_lib vndk_core vndk_indirect vndk_fwk_ext vndk_vnd_ext') - -def print_vndk_lib(vndk_lib, file=sys.stdout): - # SP-NDK and SP-HAL - print_sp_lib(vndk_lib, file=file) - - # VNDK (framework) - for lib in sorted_lib_path_list(vndk_lib.vndk_core): - print('vndk-core:', lib, file=file) - for lib in sorted_lib_path_list(vndk_lib.vndk_indirect): - print('vndk-indirect:', lib, file=file) - for lib in sorted_lib_path_list(vndk_lib.vndk_fwk_ext): - print('vndk-fwk-ext:', lib, file=file) - - # VNDK (vendor) - for lib in sorted_lib_path_list(vndk_lib.vndk_vnd_ext): - print('vndk-vnd-ext:', lib, file=file) - for lib in sorted_lib_path_list(vndk_lib.extra_vendor_lib): - print('extra-vendor-lib:', lib, file=file) - - SPLibResult = collections.namedtuple( 'SPLibResult', 'sp_hal sp_hal_dep vndk_sp_hal sp_ndk sp_ndk_indirect ' @@ -807,44 +835,53 @@ def sorted_lib_path_list(libs): libs.sort() return libs +_VNDK_RESULT_FIELD_NAMES = ( + 'll_ndk', 'll_ndk_indirect', 'sp_ndk', 'sp_ndk_indirect', + 'vndk_sp', 'vndk_sp_unused', 'vndk_sp_indirect', + 'vndk_sp_indirect_unused', 'vndk_sp_indirect_private', 'vndk', + 'vndk_indirect', 'fwk_only', 'fwk_only_rs', 'sp_hal', 'sp_hal_dep', + 'vnd_only', 'vndk_ext', 'vndk_sp_ext', 'vndk_sp_indirect_ext', + 'extra_vndk_sp_indirect') + +VNDKResult = defaultnamedtuple('VNDKResult', _VNDK_RESULT_FIELD_NAMES, set()) + + +class ELFLibDict(defaultnamedtuple('ELFLibDict', ('lib32', 'lib64'), {})): + def get_lib_dict(self, elf_class): + return self[elf_class - 1] + + def add(self, path, lib): + self.get_lib_dict(lib.elf.ei_class)[path] = lib + + def remove(self, lib): + del self.get_lib_dict(lib.elf.ei_class)[lib.path] + + def get(self, path, default=None): + for lib_set in self: + res = lib_set.get(path, None) + if res: + return res + return default + + def keys(self): + return itertools.chain(self.lib32.keys(), self.lib64.keys()) + + def values(self): + return itertools.chain(self.lib32.values(), self.lib64.values()) + + def items(self): + return itertools.chain(self.lib32.items(), self.lib64.items()) + class ELFLinker(object): - LIB32_SEARCH_PATH = ( - '/system/lib', - '/system/lib/vndk', - '/system/lib/vndk-ext', - '/vendor/lib', - ) - - LIB64_SEARCH_PATH = ( - '/system/lib64', - '/system/lib64/vndk', - '/system/lib64/vndk-ext', - '/vendor/lib64', - ) - - def __init__(self): - self.lib32 = dict() - self.lib64 = dict() - self.lib_pt = [dict() for i in range(NUM_PARTITIONS)] - - self.lib32_resolver = ELFResolver(self.lib32, self.LIB32_SEARCH_PATH) - self.lib64_resolver = ELFResolver(self.lib64, self.LIB64_SEARCH_PATH) + self.lib_pt = [ELFLibDict() for i in range(NUM_PARTITIONS)] def _add_lib_to_lookup_dict(self, lib): - if lib.elf.is_32bit: - self.lib32[lib.path] = lib - else: - self.lib64[lib.path] = lib - self.lib_pt[lib.partition][lib.path] = lib + self.lib_pt[lib.partition].add(lib.path, lib) def _remove_lib_from_lookup_dict(self, lib): - if lib.elf.is_32bit: - del self.lib32[lib.path] - else: - del self.lib64[lib.path] - del self.lib_pt[lib.partition][lib.path] + self.lib_pt[lib.partition].remove(lib) def add_lib(self, partition, path, elf): lib = ELFLinkData(partition, path, elf) @@ -858,32 +895,52 @@ class ELFLinker(object): self._add_lib_to_lookup_dict(lib) def add_dep(self, src_path, dst_path, ty): - for lib_set in (self.lib32, self.lib64): - src = lib_set.get(src_path) - dst = lib_set.get(dst_path) + for elf_class in (ELF.ELFCLASS32, ELF.ELFCLASS64): + src = self.get_lib_in_elf_class(elf_class, src_path) + dst = self.get_lib_in_elf_class(elf_class, dst_path) if src and dst: src.add_dep(dst, ty) return print('error: cannot add dependency from {} to {}.' .format(src_path, dst_path), file=sys.stderr) + def get_lib_in_elf_class(self, elf_class, path, default=None): + for partition in range(NUM_PARTITIONS): + res = self.lib_pt[partition].get_lib_dict(elf_class).get(path) + if res: + return res + return default + def get_lib(self, path): - for lib_set in (self.lib32, self.lib64): + for lib_set in self.lib_pt: lib = lib_set.get(path) if lib: return lib return None - def get_libs(self, paths, report_error): + def get_libs(self, paths, report_error=None): result = set() for path in paths: lib = self.get_lib(path) if not lib: + if report_error is None: + raise ValueError('path not found ' + path) report_error(path) continue result.add(lib) return result + def all_libs(self): + for lib_set in self.lib_pt: + for lib in lib_set.values(): + yield lib + + def _compute_lib_dict(self, elf_class): + res = dict() + for lib_pt in self.lib_pt: + res.update(lib_pt.get_lib_dict(elf_class)) + return res + @staticmethod def _compile_path_matcher(root, subdirs): dirs = [os.path.normpath(os.path.join(root, i)) for i in subdirs] @@ -965,27 +1022,84 @@ class ELFLinker(object): self._resolve_lib_imported_symbols(lib, imported_libs, generic_refs) def _resolve_lib_set_deps(self, lib_set, resolver, generic_refs): - for lib in lib_set.values(): + for lib in lib_set: self._resolve_lib_deps(lib, resolver, generic_refs) - def resolve_deps(self, generic_refs=None): - self._resolve_lib_set_deps( - self.lib32, self.lib32_resolver, generic_refs) - self._resolve_lib_set_deps( - self.lib64, self.lib64_resolver, generic_refs) + SYSTEM_SEARCH_PATH = ( + '/system/${LIB}', + '/vendor/${LIB}', + ) - def all_lib(self): - for lib_set in self.lib_pt: - for lib in lib_set.values(): - yield lib + VENDOR_SEARCH_PATH = ( + '/vendor/${LIB}', + '/vendor/${LIB}/vndk-sp', + '/system/${LIB}/vndk-sp', + '/system/${LIB}', # For degenerated VNDK libs. + ) + + VNDK_SP_SEARCH_PATH = ( + '/vendor/${LIB}/vndk-sp', + '/system/${LIB}/vndk-sp', + '/vendor/${LIB}', # To discover missing vndk-sp dependencies. + '/system/${LIB}', # To discover missing vndk-sp dependencies. + ) + + @staticmethod + def _subst_search_path(search_path, elf_class): + lib_dir_name = 'lib' if elf_class == ELF.ELFCLASS32 else 'lib64' + return [path.replace('${LIB}', lib_dir_name) for path in search_path] + + @staticmethod + def _is_in_vndk_sp_dir(path): + return os.path.basename(os.path.dirname(path)).startswith('vndk-sp') + + def _resolve_elf_class_deps(self, elf_class, generic_refs): + system_lib_dict = self.lib_pt[PT_SYSTEM].get_lib_dict(elf_class) + vendor_lib_dict = self.lib_pt[PT_VENDOR].get_lib_dict(elf_class) + lib_dict = self._compute_lib_dict(elf_class) + + # Resolve system libs. + system_libs = [lib for lib in system_lib_dict.values() + if not self._is_in_vndk_sp_dir(lib.path)] + search_path = self._subst_search_path( + self.SYSTEM_SEARCH_PATH, elf_class) + resolver = ELFResolver(lib_dict, search_path) + self._resolve_lib_set_deps(system_libs, resolver, generic_refs) + + # Resolve vendor libs. + vendor_libs = [lib for lib in vendor_lib_dict.values() + if not self._is_in_vndk_sp_dir(lib.path)] + search_path = self._subst_search_path( + self.VENDOR_SEARCH_PATH, elf_class) + resolver = ELFResolver(lib_dict, search_path) + self._resolve_lib_set_deps(vendor_libs, resolver, generic_refs) + + # Resolve vndk-sp libs + vndk_sp = [lib for lib in lib_dict.values() + if self._is_in_vndk_sp_dir(lib.path)] + search_path = self._subst_search_path( + self.VNDK_SP_SEARCH_PATH, elf_class) + resolver = ELFResolver(lib_dict, search_path) + self._resolve_lib_set_deps(vndk_sp, resolver, generic_refs) + + def resolve_deps(self, generic_refs=None): + self._resolve_elf_class_deps(ELF.ELFCLASS32, generic_refs) + self._resolve_elf_class_deps(ELF.ELFCLASS64, generic_refs) def compute_path_matched_lib(self, path_patterns): patt = re.compile('|'.join('(?:' + p + ')' for p in path_patterns)) - return set(lib for lib in self.all_lib() if patt.match(lib.path)) + return set(lib for lib in self.all_libs() if patt.match(lib.path)) + + def compute_predefined_fwk_only_rs(self): + """Find all fwk-only-rs libraries.""" + path_patterns = ( + '^/system/lib(?:64)?/libft2\\.so$', + '^/system/lib(?:64)?/libmediandk\\.so', + ) + return self.compute_path_matched_lib(path_patterns) def compute_predefined_vndk_sp(self): """Find all vndk-sp libraries.""" - path_patterns = ( # Visible to SP-HALs '^.*/android\\.hardware\\.graphics\\.allocator@2\\.0\\.so$', @@ -1012,14 +1126,12 @@ class ELFLinker(object): ) return self.compute_path_matched_lib(path_patterns) - def compute_predefined_vndk_sp_indirect(self): """Find all vndk-sp-indirect libraries.""" path_patterns = ( # Invisible to SP-HALs '^.*/libbacktrace\\.so$', '^.*/libblas\\.so$', - '^.*/libft2\\.so$', '^.*/liblzma\\.so$', '^.*/libpng\\.so$', '^.*/libunwind\\.so$', @@ -1049,7 +1161,7 @@ class ELFLinker(object): def compute_sp_ndk(self): """Find all SP-NDK libraries.""" - return set(lib for lib in self.all_lib() if lib.is_sp_ndk) + return set(lib for lib in self.all_libs() if lib.is_sp_ndk) def compute_sp_lib(self, generic_refs): def is_ndk(lib): @@ -1103,9 +1215,8 @@ class ELFLinker(object): return self._po_sorted(lib_set, lambda x: x.users) def normalize_partition_tags(self, sp_hals, generic_refs): - system_libs_po = self._deps_po_sorted(self.lib_pt[PT_SYSTEM].values()) - system_libs = self.lib_pt[PT_SYSTEM] - vendor_libs = self.lib_pt[PT_VENDOR] + system_libs = set(self.lib_pt[PT_SYSTEM].values()) + system_libs_po = self._deps_po_sorted(system_libs) def is_system_lib_or_sp_hal(lib): return lib.is_system_lib() or lib in sp_hals @@ -1142,346 +1253,187 @@ class ELFLinker(object): 'vendor partition.' .format(lib.path, dep.path, lib.path), file=sys.stderr) - lib.partition = PT_VENDOR - vendor_libs[lib.path] = lib - del system_libs[lib.path] + new_path = lib.path.replace('/system/', '/vendor/') + self.rename_lib(lib, PT_VENDOR, new_path) - def find_existing_vndk(self): - def collect_libs_with_path_pattern(pattern): - result = set() - pattern = re.compile(pattern) - for lib_set in (self.lib32.values(), self.lib64.values()): - for lib in lib_set: - if pattern.match(lib.path): - result.add(lib) - return result + def compute_degenerated_vndk(self, sp_lib, generic_refs, + tagged_paths=None, + action_ineligible_vndk_sp='follow,warn', + action_ineligible_vndk='follow,warn'): + # Find LL-NDK and SP-NDK libs. + ll_ndk = set(lib for lib in self.all_libs() if lib.is_ll_ndk) - vndk_core = collect_libs_with_path_pattern( - '^/system/lib(?:64)?/vndk(?:-\\d+)?/') - vndk_fwk_ext = collect_libs_with_path_pattern( - '^/system/lib(?:64)?/vndk(?:-\\d+)?-ext?/') - vndk_vnd_ext = collect_libs_with_path_pattern( - '^/vendor/lib(?:64)?/vndk(?:-\\d+)?-ext?/') + def is_not_ll_ndk_indirect(lib): + return lib.is_ll_ndk - return (vndk_core, vndk_fwk_ext, vndk_vnd_ext) + ll_ndk_indirect = self.compute_closure(ll_ndk, is_not_ll_ndk_indirect) + ll_ndk_indirect -= ll_ndk - def compute_vndk(self, vndk_customized_for_system, - vndk_customized_for_vendor, generic_refs, banned_libs): - return self._compute_vndk( - self.compute_sp_lib(generic_refs), vndk_customized_for_system, - vndk_customized_for_vendor, generic_refs, banned_libs) + sp_ndk = set(lib for lib in self.all_libs() if lib.is_sp_ndk) - def _compute_vndk(self, sp_lib, vndk_customized_for_system, - vndk_customized_for_vendor, generic_refs, banned_libs): - # Compute sp-hal and vndk-sp. - vndk_sp = sp_lib.vndk_sp_hal | sp_lib.sp_ndk_indirect | \ - sp_lib.vndk_sp_both - sp_hal_closure = sp_lib.sp_hal | sp_lib.sp_hal_dep + def is_not_sp_ndk_indirect(lib): + return lib.is_ll_ndk or lib.is_sp_ndk or lib in ll_ndk_indirect + + sp_ndk_indirect = self.compute_closure(sp_ndk, is_not_sp_ndk_indirect) + sp_ndk_indirect -= sp_ndk + + # Find SP-HAL libs. + sp_hal = self.compute_predefined_sp_hal() # Normalize partition tags. We expect many violations from the # pre-Treble world. Guess a resolution for the incorrect partition # tag. - self.normalize_partition_tags(sp_lib.sp_hal, generic_refs) + self.normalize_partition_tags(sp_hal, generic_refs) - # ELF resolvers. - VNDK_CORE_SEARCH_PATH32 = ( - '/system/lib/vndk', - '/system/lib', - ) + # Find SP-HAL-Dep libs. + def is_not_sp_hal_dep(lib): + if lib.is_ll_ndk or lib.is_sp_ndk or lib in sp_hal: + return True + if not generic_refs: + # Use simple heuristic when generic reference is not available. + return lib.partition == PT_SYSTEM + return generic_refs.has_same_name_lib(lib) - VNDK_CORE_SEARCH_PATH64 = ( - '/system/lib64/vndk', - '/system/lib64', - ) + sp_hal_dep = self.compute_closure(sp_hal, is_not_sp_hal_dep) + sp_hal_dep -= sp_hal - VENDOR_SEARCH_PATH32 = ( - '/system/lib/vndk', - '/vendor/lib', + # Find FWK-ONLY-RS libs. + fwk_only_rs = self.compute_predefined_fwk_only_rs() - # FIXME: Remove following line after we fixed vndk-sp resolution. - '/system/lib', - ) + # Find VNDK-SP libs. + def is_not_vndk_sp(lib): + return lib.is_ll_ndk or lib.is_sp_ndk or lib in sp_hal or \ + lib in sp_hal_dep - VENDOR_SEARCH_PATH64 = ( - '/system/lib64/vndk', - '/vendor/lib64', - - # FIXME: Remove following line after we fixed vndk-sp resolution. - '/system/lib64', - ) - - vndk_core_resolver32 = ELFResolver(self.lib32, VNDK_CORE_SEARCH_PATH32) - vndk_core_resolver64 = ELFResolver(self.lib64, VNDK_CORE_SEARCH_PATH64) - vendor_resolver32 = ELFResolver(self.lib32, VENDOR_SEARCH_PATH32) - vendor_resolver64 = ELFResolver(self.lib64, VENDOR_SEARCH_PATH64) - - # Collect existing VNDK libraries. - vndk_core, vndk_fwk_ext, vndk_vnd_ext = self.find_existing_vndk() - - # Collect VNDK candidates. - def is_not_vndk(lib): - return (lib.is_ndk or banned_libs.is_banned(lib.path) or - (lib in sp_hal_closure) or (lib in vndk_sp)) - - def collect_libs_with_partition_user(lib_set, partition): - result = set() - for lib in lib_set: - if is_not_vndk(lib): + action_ineligible_vndk_sp = set(action_ineligible_vndk_sp.split(',')) + predefined_vndk_sp = self.compute_predefined_vndk_sp() + vndk_sp = set() + for lib in itertools.chain(sp_hal, sp_hal_dep): + for dep in lib.deps: + if is_not_vndk_sp(dep): continue - if any(user.partition == partition for user in lib.users): - result.add(lib) - return result + if dep in predefined_vndk_sp: + vndk_sp.add(dep) + continue + if 'warn' in action_ineligible_vndk_sp: + print('error: SP-HAL {} depends on non vndk-sp ' + 'library {}.'.format(lib.path, dep.path), + file=sys.stderr) + if 'follow' in action_ineligible_vndk_sp: + vndk_sp.add(dep) - vndk_candidates = collect_libs_with_partition_user( - self.lib_pt[PT_SYSTEM].values(), PT_VENDOR) + # Add other predefined VNDK-SP even if they are not actually used by + # SP-HAL libs. + vndk_sp_unused = set(lib for lib in predefined_vndk_sp + if self._is_in_vndk_sp_dir(lib.path)) - vndk_sp - vndk_visited = set(vndk_candidates) + # Find VNDK-SP-Indirect libs. + def is_not_vndk_sp_indirect(lib): + return lib.is_ll_ndk or lib.is_sp_ndk or lib in vndk_sp or \ + lib in fwk_only_rs - # Sets for missing libraries. - extra_vendor_lib = set() + vndk_sp_indirect = self.compute_closure( + vndk_sp, is_not_vndk_sp_indirect) + vndk_sp_indirect -= vndk_sp - def get_vndk_core_lib_name(lib): - lib_name = os.path.basename(lib.path) - lib_dir_name = 'lib' if lib.elf.is_32bit else 'lib64' - return os.path.join('/system', lib_dir_name, 'vndk', lib_name) + def is_not_vndk_sp_indirect_unused(lib): + return is_not_vndk_sp_indirect(lib) or lib in vndk_sp_indirect + vndk_sp_indirect_unused = self.compute_closure( + vndk_sp_unused, is_not_vndk_sp_indirect_unused) + vndk_sp_indirect_unused -= vndk_sp_unused - def get_vndk_fwk_ext_lib_name(lib): - lib_name = os.path.basename(lib.path) - lib_dir_name = 'lib' if lib.elf.is_32bit else 'lib64' - return os.path.join('/system', lib_dir_name, 'vndk-ext', lib_name) + # TODO: Compute VNDK-SP-Indirect-Private. + vndk_sp_indirect_private = set() - def get_vndk_vnd_ext_lib_name(lib): - lib_name = os.path.basename(lib.path) - lib_dir_name = 'lib' if lib.elf.is_32bit else 'lib64' - return os.path.join('/vendor', lib_dir_name, 'vndk-ext', lib_name) + predefined_vndk_sp_indirect = self.compute_predefined_vndk_sp_indirect() - def is_valid_vndk_core_dep(path): - d = os.path.dirname(path) - return (d == '/system/lib' or d == '/system/lib64' or - d == '/system/lib/vndk' or d == '/system/lib64/vndk') + # TODO: Compute VNDK-SP-Ext and VNDK-SP-Indirect-Ext. + vndk_sp_ext = set() - def add_generic_lib_to_vndk_core(lib): - """Add a library to vndk-core.""" - elf = generic_refs.refs[lib.path] + def is_not_vndk_sp_indirect_ext(lib): + return lib.is_ll_ndk or lib.is_sp_ndk or lib in vndk_sp_ext or \ + lib in predefined_vndk_sp or \ + lib in predefined_vndk_sp_indirect - # Create new vndk-core lib from generic reference. - vndk_core_lib_path = get_vndk_core_lib_name(lib) - vndk_core_lib = self.add_lib(PT_SYSTEM, vndk_core_lib_path, elf) + vndk_sp_indirect_ext = self.compute_closure( + vndk_sp_ext, is_not_vndk_sp_indirect_ext) + vndk_sp_indirect_ext -= vndk_sp_ext - # Resovle the library dependencies. - resolver = vndk_core_resolver32 if lib.elf.is_32bit else \ - vndk_core_resolver64 - self._resolve_lib_deps(vndk_core_lib, resolver, generic_refs) + extra_vndk_sp_indirect = vndk_sp - predefined_vndk_sp - \ + predefined_vndk_sp_indirect - assert all(is_valid_vndk_core_dep(dep.path) - for dep in vndk_core_lib.deps) + # Find VNDK libs (a.k.a. system shared libs directly used by vendor + # partition.) + def is_not_vndk(lib): + if lib.is_ll_ndk or lib.is_sp_ndk or lib in vndk_sp: + return True + if lib.partition != PT_SYSTEM: + return True + if not generic_refs: + # If generic reference is not available, we assume all system + # libs are eligible vndk. + return False + return not generic_refs.has_same_name_lib(lib) - # Add vndk-core to the set. - vndk_core.add(vndk_core_lib) - return vndk_core_lib - - def add_to_vndk_core(lib): - self.rename_lib(lib, PT_SYSTEM, get_vndk_core_lib_name(lib)) - vndk_core.add(lib) - - # Compute vndk-core, vndk-fwk-ext and vndk-vnd-ext. - if not generic_refs: - for lib in vndk_candidates: - add_to_vndk_core(lib) - else: - while vndk_candidates: - if __debug__: - # Loop invariant: These set should be pairwise independent. - # Each VNDK libraries should have their ELFLinkData - # instance. - assert not (vndk_core & vndk_fwk_ext) - assert not (vndk_core & vndk_vnd_ext) - assert not (vndk_fwk_ext & vndk_vnd_ext) - - # Loop invariant: The library names in vndk_fwk_ext and - # vndk_vnd_ext must exist in vndk_core as well. - vndk_core_lib_names = \ - set(os.path.basename(x.path) for x in vndk_core) - vndk_fwk_ext_lib_names = \ - set(os.path.basename(x.path) for x in vndk_fwk_ext) - vndk_vnd_ext_lib_names = \ - set(os.path.basename(x.path) for x in vndk_vnd_ext) - assert vndk_fwk_ext_lib_names <= vndk_core_lib_names - assert vndk_vnd_ext_lib_names <= vndk_core_lib_names - - prev_vndk_candidates = vndk_candidates - vndk_candidates = set() - - def replace_linked_lib(user, old_lib, new_lib, dep_type): - user.remove_dep(old_lib, dep_type) - user.add_dep(new_lib, dep_type) - for symbol, imported_lib in user.linked_symbols.items(): - if imported_lib == old_lib: - user.linked_symbols[symbol] = new_lib - - def replace_generic_lib_usages(lib, generic_lib): - for user, dep_type in list(lib.users_with_type): - if lib not in user.imported_ext_symbols: - replace_linked_lib(user, lib, generic_lib, dep_type) - - def add_to_vndk_fwk_ext(lib, generic_lib): - self.rename_lib(lib, PT_SYSTEM, - get_vndk_fwk_ext_lib_name(lib)) - vndk_fwk_ext.add(lib) - replace_generic_lib_usages(lib, generic_lib) - - def add_to_vndk_vnd_ext(lib, generic_lib): - """Add a library to vndk-vnd-ext.""" - - replace_generic_lib_usages(lib, generic_lib) - - # Create a new vndk-vnd-ext library. - vndk_vnd_ext_lib = self.add_lib( - PT_VENDOR, get_vndk_vnd_ext_lib_name(lib), lib.elf) - - # Vendor libraries should link to vndk_vnd_ext_lib instead. - for user, dep_type in list(lib.users_with_type): - if not user.is_system_lib(): - replace_linked_lib(user, lib, vndk_vnd_ext_lib, - dep_type) - - # Resolve the dependencies. In order to find more - # dependencies from vendor partition to system partition, - # continue to resolve dependencies with the global - # ELFResolver. - resolver = self.lib32_resolver if lib.elf.is_32bit else \ - self.lib64_resolver - self._resolve_lib_deps(vndk_vnd_ext_lib, resolver, - generic_refs) - - add_deps_to_vndk_candidate(vndk_vnd_ext_lib) - - vndk_vnd_ext.add(vndk_vnd_ext_lib) - - def add_to_vndk_candidate(lib): - if is_not_vndk(lib): - return - if lib not in vndk_visited: - vndk_candidates.add(lib) - vndk_visited.add(lib) - - def add_deps_to_vndk_candidate(lib): - for dep in lib.deps: - if dep.is_system_lib(): - add_to_vndk_candidate(dep) - - # Remove non-AOSP libraries. - vndk_extended_candidates = set() - vndk_customized_candidates = set() - for lib in prev_vndk_candidates: - category = generic_refs.classify_lib(lib) - if category == GenericRefs.NEW_LIB: - extra_vendor_lib.add(lib) - add_deps_to_vndk_candidate(lib) - elif category == GenericRefs.EXPORT_EQUAL: - vndk_customized_candidates.add(lib) - elif category == GenericRefs.EXPORT_SUPER_SET: - vndk_extended_candidates.add(lib) - else: - print('error: {}: vndk library must not be modified.' - .format(lib.path), file=sys.stderr) - - # Classify VNDK customized candidates. - for lib in vndk_customized_candidates: - if not lib.imported_ext_symbols: - # Inward-customized VNDK-core libraries. - add_to_vndk_core(lib) - else: - # Outward-customized VNDK libraries. - generic_lib = add_generic_lib_to_vndk_core(lib) - if lib in vndk_customized_for_system: - add_to_vndk_fwk_ext(lib, generic_lib) - if lib in vndk_customized_for_vendor: - add_to_vndk_vnd_ext(lib, generic_lib) - - # Compute VNDK extension candidates. - for lib in self._users_po_sorted(vndk_extended_candidates): - # Check the users of the extended exported symbols. - has_system_users = False - has_vendor_users = False - for user in lib.users: - if lib in user.imported_ext_symbols: - if user.is_system_lib(): - has_system_users = True - else: - has_vendor_users = True - if has_system_users and has_vendor_users: - break - - generic_lib = add_generic_lib_to_vndk_core(lib) - if has_system_users: - add_to_vndk_fwk_ext(lib, generic_lib) - if has_vendor_users: - add_to_vndk_vnd_ext(lib, generic_lib) - - # Compute the closure of the VNDK libs. - visited_libs = set(vndk_core) - processed_paths = set(lib.path for lib in vndk_core) - stack = [] - - def add_vndk_core_deps_to_stack(lib): + action_ineligible_vndk = set(action_ineligible_vndk.split(',')) + vndk = set() + for lib in self.lib_pt[PT_VENDOR].values(): for dep in lib.deps: if is_not_vndk(dep): continue - if dep not in visited_libs: - stack.append(dep) - visited_libs.add(dep) + if not tagged_paths or \ + tagged_paths.is_path_visible(lib.path, dep.path): + vndk.add(dep) + continue + if 'warn' in action_ineligible_vndk: + print('warning: vendor lib/exe {} depends on ineligible ' + 'framework shared lib {}.' + .format(lib.path, dep.path), file=sys.stderr) + if 'follow' in action_ineligible_vndk: + vndk.add(dep) - for lib in vndk_core: - add_vndk_core_deps_to_stack(lib) + vndk_indirect = self.compute_closure(vndk, is_not_vndk) + vndk_indirect -= vndk_indirect - while stack: - lib = stack.pop() + # Compute the extended usages from vendor partition. + # FIXME: DAUX libraries won't be found by the following algorithm. + vndk_ext = set() - vndk_lib_path = get_vndk_core_lib_name(lib) - if vndk_lib_path in processed_paths: - continue + def collect_vndk_ext(libs): + result = set() + for lib in libs: + for dep in lib.imported_ext_symbols: + if dep in vndk and dep not in vndk_ext: + result.add(dep) + return result - processed_paths.add(vndk_lib_path) + candidates = collect_vndk_ext(self.lib_pt[PT_VENDOR].values()) - if lib.imported_ext_symbols or \ - (generic_refs and not generic_refs.is_equivalent_lib(lib)): - generic_lib = add_generic_lib_to_vndk_core(lib) - add_vndk_core_deps_to_stack(generic_lib) - add_to_vndk_fwk_ext(lib, generic_lib) - else: - add_to_vndk_core(lib) - add_vndk_core_deps_to_stack(lib) - - # Truncate all vendor libs and resolve it again. - for lib in self.lib_pt[PT_VENDOR].values(): - lib._deps = (set(), set()) - lib._users = (set(), set()) - lib.imported_ext_symbols = collections.defaultdict(set) - lib.unresolved_symbols = set() - lib.linked_symbols = dict() - - for lib in self.lib_pt[PT_VENDOR].values(): - resolver = vendor_resolver32 if lib.elf.is_32bit else \ - vendor_resolver64 - self._resolve_lib_deps(lib, resolver, generic_refs) - - # Separate vndk-core and vndk-indirect. - vndk_core_indirect = vndk_core - vndk_core = set() - vndk_indirect = set() - for lib in vndk_core_indirect: - if any(not user.is_system_lib() for user in lib.users): - vndk_core.add(lib) - else: - vndk_indirect.add(lib) + while candidates: + vndk_ext |= candidates + candidates = collect_vndk_ext(candidates) return VNDKResult( - sp_lib.sp_hal, sp_lib.sp_hal_dep, sp_lib.vndk_sp_hal, - sp_lib.sp_ndk, sp_lib.sp_ndk_indirect, - sp_lib.vndk_sp_both, - extra_vendor_lib, vndk_core, vndk_indirect, - vndk_fwk_ext, vndk_vnd_ext) + ll_ndk=ll_ndk, + ll_ndk_indirect=ll_ndk_indirect, + sp_ndk=sp_ndk, + sp_ndk_indirect=sp_ndk_indirect, + vndk_sp=vndk_sp, + vndk_sp_indirect=vndk_sp_indirect, + # vndk_sp_indirect_private=vndk_sp_indirect_private, + vndk_sp_unused=vndk_sp_unused, + vndk_sp_indirect_unused=vndk_sp_indirect_unused, + vndk=vndk, + vndk_indirect=vndk_indirect, + # fwk_only=fwk_only, + fwk_only_rs=fwk_only_rs, + sp_hal=sp_hal, + sp_hal_dep=sp_hal_dep, + # vnd_only=vnd_only, + vndk_ext=vndk_ext, + # vndk_sp_ext=vndk_sp_ext, + # vndk_sp_indirect_ext=vndk_sp_indirect_ext, + extra_vndk_sp_indirect=extra_vndk_sp_indirect) def compute_vndk_cap(self, banned_libs): # ELF files on vendor partitions are banned unconditionally. ELF files @@ -1588,9 +1540,11 @@ class GenericRefs(object): def __init__(self): self.refs = dict() + self._lib_names = set() - def add(self, name, elf): - self.refs[name] = elf + def add(self, path, elf): + self.refs[path] = elf + self._lib_names.add(os.path.basename(path)) def _load_from_sym_dir(self, root): root = os.path.abspath(root) @@ -1600,9 +1554,9 @@ class GenericRefs(object): if not filename.endswith('.sym'): continue path = os.path.join(base, filename) - lib_name = '/' + path[prefix_len:-4] + lib_path = '/' + path[prefix_len:-4] with open(path, 'r') as f: - self.add(lib_name, ELF.load_dump(path)) + self.add(lib_path, ELF.load_dump(path)) @staticmethod def create_from_sym_dir(root): @@ -1636,6 +1590,9 @@ class GenericRefs(object): def is_equivalent_lib(self, lib): return self.classify_lib(lib) == GenericRefs.EXPORT_EQUAL + def has_same_name_lib(self, lib): + return os.path.basename(lib.path) in self._lib_names + #------------------------------------------------------------------------------ # Commands @@ -1740,21 +1697,8 @@ class VNDKCommandBase(ELFGraphCommand): def add_argparser_options(self, parser): super(VNDKCommandBase, self).add_argparser_options(parser) - parser.add_argument( - '--ban-vendor-lib-dep', action='append', - help='library that must not be used by vendor binaries') - - parser.add_argument( - '--outward-customization-default-partition', default='system', - help='default partition for outward customized vndk libs') - - parser.add_argument( - '--outward-customization-for-system', action='append', - help='outward customized vndk for system partition') - - parser.add_argument( - '--outward-customization-for-vendor', action='append', - help='outward customized vndk for vendor partition') + parser.add_argument('--no-default-dlopen-deps', action='store_true', + help='do not add default dlopen dependencies') def _check_arg_dir_exists(self, arg_name, dirs): for path in dirs: @@ -1771,55 +1715,25 @@ class VNDKCommandBase(ELFGraphCommand): self._check_arg_dir_exists('--system', args.system) self._check_arg_dir_exists('--vendor', args.vendor) - def _get_banned_libs_from_args(self, args): - if not args.ban_vendor_lib_dep: - return BannedLibDict.create_default() - - banned_libs = BannedLibDict() - for name in args.ban_vendor_lib_dep: - banned_libs.add(name, 'user-banned', BA_WARN) - return banned_libs - - def _get_outward_customized_sets_from_args(self, args, graph): - vndk_customized_for_system = set() - vndk_customized_for_vendor = set() - system_libs = graph.lib_pt[PT_SYSTEM].values() - - if args.outward_customization_default_partition in {'system', 'both'}: - vndk_customized_for_system.update(system_libs) - - if args.outward_customization_default_partition in {'vendor', 'both'}: - vndk_customized_for_vendor.update(system_libs) - - if args.outward_customization_for_system: - vndk_customized_for_system.update( - graph.get_libs( - args.outward_customization_for_system, lambda x: None)) - - if args.outward_customization_for_vendor: - vndk_customized_for_vendor.update( - graph.get_libs( - args.outward_customization_for_vendor, lambda x: None)) - return (vndk_customized_for_system, vndk_customized_for_vendor) - def create_from_args(self, args): """Create all essential data structures for VNDK computation.""" self.check_dirs_from_args(args) generic_refs = self.get_generic_refs_from_args(args) - banned_libs = self._get_banned_libs_from_args(args) graph = ELFLinker.create(args.system, args.system_dir_as_vendor, args.vendor, args.vendor_dir_as_system, args.load_extra_deps, generic_refs=generic_refs) - vndk_customized_for_system, vndk_customized_for_vendor = \ - self._get_outward_customized_sets_from_args(args, graph) + if not args.no_default_dlopen_deps: + script_dir = os.path.dirname(os.path.abspath(__file__)) + minimum_dlopen_deps = os.path.join(script_dir, 'datasets', + 'minimum_dlopen_deps.txt') + graph.load_extra_deps(minimum_dlopen_deps) - return (generic_refs, banned_libs, graph, vndk_customized_for_system, - vndk_customized_for_vendor) + return (generic_refs, graph) class VNDKCommand(VNDKCommandBase): @@ -1834,6 +1748,20 @@ class VNDKCommand(VNDKCommandBase): '--warn-incorrect-partition', action='store_true', help='warn about libraries only have cross partition linkages') + parser.add_argument( + '--full', action='store_true', + help='print all classification') + + parser.add_argument('--tag-file', help='lib tag file') + + parser.add_argument( + '--action-ineligible-vndk-sp', default='warn,follow', + help='action when a sp-hal uses non-vndk-sp libs') + + parser.add_argument( + '--action-ineligible-vndk', default='warn,follow', + help='action when a vendor lib/exe uses fwk-only libs') + def _warn_incorrect_partition_lib_set(self, lib_set, partition, error_msg): for lib in lib_set.values(): if not lib.num_users: @@ -1853,15 +1781,14 @@ class VNDKCommand(VNDKCommandBase): 'usages.') def _check_ndk_extensions(self, graph, generic_refs): - for lib_set in (graph.lib32, graph.lib64): + for lib_set in graph.lib_pt: for lib in lib_set.values(): if lib.is_ndk and not generic_refs.is_equivalent_lib(lib): print('warning: {}: NDK library should not be extended.' .format(lib.path), file=sys.stderr) def main(self, args): - generic_refs, banned_libs, graph, vndk_customized_for_system, \ - vndk_customized_for_vendor = self.create_from_args(args) + generic_refs, graph = self.create_from_args(args) # Check the API extensions to NDK libraries. if generic_refs: @@ -1870,13 +1797,25 @@ class VNDKCommand(VNDKCommandBase): if args.warn_incorrect_partition: self._warn_incorrect_partition(graph) + if args.tag_file: + tagged_paths = TaggedPathDict.create_from_csv_path(args.tag_file) + else: + tagged_paths = None + # Compute vndk heuristics. - vndk_lib = graph.compute_vndk(vndk_customized_for_system, - vndk_customized_for_vendor, generic_refs, - banned_libs) + sp_lib = graph.compute_sp_lib(generic_refs) + vndk_lib = graph.compute_degenerated_vndk( + sp_lib, generic_refs, tagged_paths, + args.action_ineligible_vndk_sp, args.action_ineligible_vndk) # Print results. - print_vndk_lib(vndk_lib) + field_names = ['extra_vndk_sp_indirect', 'vndk_ext'] + if args.full: + field_names = _VNDK_RESULT_FIELD_NAMES + for field_name in field_names: + tag = field_name + ':' + for lib in sorted_lib_path_list(getattr(vndk_lib, field_name)): + print(tag, lib) return 0 @@ -1892,19 +1831,17 @@ class DepsInsightCommand(VNDKCommandBase): '--output', '-o', help='output directory') def main(self, args): - generic_refs, banned_libs, graph, vndk_customized_for_system, \ - vndk_customized_for_vendor = self.create_from_args(args) + generic_refs, graph = self.create_from_args(args) # Compute vndk heuristics. - vndk_lib = graph.compute_vndk(vndk_customized_for_system, - vndk_customized_for_vendor, generic_refs, - banned_libs) + sp_lib = graph.compute_sp_lib(generic_refs) + vndk_lib = graph.compute_degenerated_vndk(sp_lib, generic_refs) # Serialize data. strs = [] strs_dict = dict() - libs = list(graph.lib32.values()) + list(graph.lib64.values()) + libs = list(graph.all_libs()) libs.sort(key=lambda lib: lib.path) libs_dict = {lib: i for i, lib in enumerate(libs)} @@ -1945,35 +1882,9 @@ class DepsInsightCommand(VNDKCommandBase): def collect_tags(lib): tags = [] - if lib.is_ll_ndk: - tags.append(get_str_idx('ll-ndk')) - if lib.is_sp_ndk: - tags.append(get_str_idx('sp-ndk')) - if lib.is_hl_ndk: - tags.append(get_str_idx('hl-ndk')) - - if lib in vndk_lib.sp_hal: - tags.append(get_str_idx('sp-hal')) - if lib in vndk_lib.sp_hal_dep: - tags.append(get_str_idx('sp-hal-dep')) - if lib in vndk_lib.vndk_sp_hal: - tags.append(get_str_idx('vndk-sp-hal')) - - if lib in vndk_lib.sp_ndk_indirect: - tags.append(get_str_idx('sp-ndk-indirect')) - if lib in vndk_lib.vndk_sp_both: - tags.append(get_str_idx('vndk-sp-both')) - - if lib in vndk_lib.vndk_core: - tags.append(get_str_idx('vndk-core')) - if lib in vndk_lib.vndk_indirect: - tags.append(get_str_idx('vndk-indirect')) - if lib in vndk_lib.vndk_fwk_ext: - tags.append(get_str_idx('vndk-fwk-ext')) - if lib in vndk_lib.vndk_vnd_ext: - tags.append(get_str_idx('vndk-vnd-ext')) - if lib in vndk_lib.extra_vendor_lib: - tags.append(get_str_idx('extra-vendor-lib')) + for field_name in _VNDK_RESULT_FIELD_NAMES: + if lib in getattr(vndk_lib, field_name): + tags.append(get_str_idx(field_name)) return tags mods = [] From 5947a9a23dbc372572bb3dd4103be36efa31447e Mon Sep 17 00:00:00 2001 From: Logan Chien Date: Thu, 8 Jun 2017 00:40:58 +0800 Subject: [PATCH 3/4] vndk-def-tool: Update README.md This commit updates readme instructions to match with the simplified VNDK in O workflow. Test: Run the described command lines with sailfish directories. Change-Id: I80f3ee4eac34fae772ba6c336e457f537cb8312c --- vndk/tools/definition-tool/README.md | 244 ++++-------------- .../templates/extra_vndk_sp_indirect.txt | 39 +++ .../definition-tool/templates/vndk_ext.txt | 37 +++ 3 files changed, 126 insertions(+), 194 deletions(-) create mode 100644 vndk/tools/definition-tool/templates/extra_vndk_sp_indirect.txt create mode 100644 vndk/tools/definition-tool/templates/vndk_ext.txt diff --git a/vndk/tools/definition-tool/README.md b/vndk/tools/definition-tool/README.md index dbda8916e..37363fc0a 100644 --- a/vndk/tools/definition-tool/README.md +++ b/vndk/tools/definition-tool/README.md @@ -1,131 +1,64 @@ VNDK Definition Tool ==================== +VNDK definition tool was designed to classify all shared libraries in the +system partition and give suggestions to copy necessary libraries to the vendor +partition. + ## Usage -Create a generic reference symbols from AOSP build: +To run VNDK definition tool, you will need three inputs: - $ python3 vndk_definition_tool.py create-generic-ref \ - -o generic_arm64/system \ - ${OUT_DIR_COMMON_BASE}/target/product/generic_arm64/system +1. The system and vendor image for your target +2. Android Treble reference image +3. Eligible VNDK list from Google (e.g. eligible-list-v3.0.csv) -Run the VNDK definition tool with: +The high-level overview of the command line usage is: - $ python3 vndk_definition_tool.py vndk \ - --system ${ANDROID_PRODUCT_OUT}/system \ - --vendor ${ANDROID_PRODUCT_OUT}/vendor \ - --load-generic-refs generic_arm64 + $ python3 ./vndk_definition_tool.py vndk \ + --system "/path/to/your/product_out/system" \ + --vendor "/path/to/your/product_out/vendor" \ + --aosp-system "/path/to/aosp/generic/system" \ + --tag-file "eligible-list-v3.0.csv" -This command will print shared libraries that belong to the following sets: +This command will print several lines such as: -1. **sp-ndk** + extra_vndk_sp_indirect: libexample1.so + extra_vndk_sp_indirect: libexample2.so + vndk_ext: libexample3.so + vndk_ext: libexample4.so - - This contains the pre-defined SP-NDK libraries. +This output implies: - - The libraries will be installed to `/system/lib[64]` +1. `libexample1.so` and `libexample2.so` should be copied into + `/vendor/lib[64]/vndk-sp`. -2. **sp-ndk-vndk-stable** - - - This contains the SP-NDK dependencies. - - - The libraries with long-term API/ABI stability/compatibility commitment. - - - The libraries will be installed to `/system/lib[64]/vndk-stable` - -3. **sp-hal** - - - This contains the pre-defined SP-HAL libraries. - - - The libraries will be installed to `/vendor/lib[64]/sameprocess` - -4. **sp-hal-dep** - - - This contains the SP-HAL non-AOSP dependencies. - - - The libraries will be installed to `/vendor/lib[64]/sameprocess` - -5. **sp-hal-vndk-stable** - - - This contains the SP-HAL AOSP dependencies. - - - The libraries with long-term API/ABI stability/compatibility commitment. - - - The libraries will be installed to `/system/lib[64]/vndk-stable` - -6. **vndk-core** - - - This contains the shared libraries used by both the framework and - vendor code. - - - The libraries must be either intact or inward-customized. - - - The libraries will be installed to `/system/lib[64]/vndk-$FWK` - -7. **vndk-indirect** - - - This contains the shared libraries which are indirectly used by - aforementioned vndk-core but not directly used by vendor code. - - - The libraries must be either intact or inward-customized. - - - The libraries will be installed to `/system/lib[64]/vndk-$FWK` - -8. **vndk-fwd-ext** - - - This contains the vndk-core/vndk-indirect overlays for *the framework*. - - - The libraries must be either outward-customized or extended. In other - words, the libraries in this list might use or define non-AOSP APIs. - - - The libraries will be installed to `/system/lib[64]/vndk-$FWK-ext` - -9. **vndk-vnd-ext** - - - This contains the vndk-core overlays for *vendor code*. - - - The libraries must be either outward-customized or extended. In other - words, the libraries in this list might use or define non-AOSP APIs. - - - The libraries will be installed to `/vendor/lib[64]/vndk-$VND-ext` - -10. **extra-vendor-lib** - - - This contains the extra libraries that have to be copied from - `/system/lib[64]` to `/vendor/lib[64]`. - - - The libraries in this list are usually the non-AOSP dependencies of - vndk-vnd-ext or other vendor code. - - - The libraries will be installed to `/vendor/lib[64]` +2. `libexample3.so` and `libexample4.so` should be copied into + `/vendor/lib[64]`. -# Sub Directory Tagging +# Boilerplates -If there are some sub directory under system partition must be treated as -vendor files, then specify such directory with: `--system-dir-as-vendor`. +There are some boilerplates in `templates` directory that can automate the +process to copy shared libraries. -Conversely, if there are some sub directory under vendor partition must be -treated as system files, then specify such directory with: -`--vendor-dir-as-system`. +If the output tagged some shared libraries with `extra_vndk_sp_indirect`, then +copy `templates/extra_vndk_sp_indirect.txt` to an Android.mk and substitute +`##_EXTRA_VNDK_SP_INDIRECT_##` with library names (without `.so`). -For example, if the device does not have an independent `vendor` partition (but -with a `vendor` folder in the `system` partition), then run this command: +If the output tagged some shared libraries with `vndk_ext`, then copy +`templates/vndk_ext.txt` to an Android.mk and substitute `##_VNDK_EXT_##` with +library names (without `.so`). - $ python3 vndk_definition_tool.py vndk \ - --system ${ANDROID_PRODUCT_OUT}/system \ - --system-dir-as-vendor vendor \ - --load-generic-refs generic_arm64 +These boilerplates only define the modules to copy shared libraries. +Developers have to add those modules to the `PRODUCT_PACKAGES` variable in +their `device.mk`. For example, in the example mentioned above, following +`PRODUCT_PACKAGES` changes are necessary for that target: -For example, if `/system/bin/hw`, `/system/lib/hw`, and `/system/lib64/hw` are -containing vendor files, then run this command: - - $ python3 vndk_definition_tool.py vndk \ - --system ${ANDROID_PRODUCT_OUT}/system \ - --system-dir-as-vendor bin/hw \ - --system-dir-as-vendor lib/hw \ - --system-dir-as-vendor lib64/hw \ - --vendor ${ANDROID_PRODUCT_OUT}/vendor \ - --load-generic-refs generic_arm64 + PRODUCT_PACKAGES += libexample1.vndk-sp-ext + PRODUCT_PACKAGES += libexample2.vndk-sp-ext + PRODUCT_PACKAGES += libexample3.vndk-ext + PRODUCT_PACKAGES += libexample4.vndk-ext ## Implicit Dependencies @@ -143,65 +76,15 @@ And then, run VNDK definition tool with: $ python3 vndk_definition_tool.py vndk \ --system ${ANDROID_PRODUCT_OUT}/system \ --vendor ${ANDROID_PRODUCT_OUT}/vendor \ - --load-generic-refs generic_arm64 \ + --aosp-system ${ANDROID_PRODUCT_OUT}/../generic_arm64_a \ + --tag-file eligible-list-v3.0.csv \ --load-extra-deps dlopen.dep -## Partition for VNDK with Outward Customization +## Remarks -An outward-customized VNDK library can be put on both system and vendor -partition. VNDK definition tool will assume such library will be installed -into /system/lib[64]/vndk-$FWK-ext by default. Use following options to change -the default behavior. - -* `--outward-customization-default-partition=[system*|vendor|both]` - - This option specifies the default destination for outward-customized VNDK - libraries. The default value is the system partition. - -* `--outward-customization-for-system=[lib]` - - This option specifies the library that should be installed to the system - partition if it is an outward-customized VNDK library. - -* `--outward-customization-for-vendor=[lib]` - - This option specifies the library that should be installed to the vendor - partition if it is an outward-customized VNDK library. - - -## Warnings - -### Incorrect Partition - -If you specify `--warn-incorrect-partition` command line option, then VNDK -definition tool will emit warnings when: - -1. A framework library is only used by vendor binaries. - -2. A vendor library is only used by framework binaries. - -This allows people to review the correct partition for the module. For example, - - warning: /system/lib/libtinyxml.so: This is a framework library with - vendor-only usages. - - warning: /system/lib64/libtinyxml.so: This is a framework library with - vendor-only usages. - -These warnings suggest that `libtinyxml.so` might be better to move to vendor -partition. - - -## Example - -We can run this against Nexus 6p Factory Image: - - $ unzip angler-nmf26f-factory-ef607244.zip - - $ cd angler-nmf26f - - $ unzip image-angler-nmf26f.zip +To run VNDK definition tool against an image (`.img`), run the following +command to mount the images and run `vndk_definition_tool.py` with `sudo`: $ simg2img system.img system.raw.img @@ -216,34 +99,7 @@ We can run this against Nexus 6p Factory Image: $ sudo mount -o loop,ro vendor.raw.img vendor $ sudo python3 vndk_definition_tool.py vndk \ - --system system --vendor vendor - -We can run this against latest Android build: - - $ python3 vndk_definition_tool.py vndk \ - --system ${ANDROID_PRODUCT_OUT}/system \ - --system-dir-as-vendor bin/hw \ - --system-dir-as-vendor lib/hw \ - --system-dir-as-vendor lib64/hw \ - --vendor ${ANDROID_PRODUCT_OUT}/vendor - - -## Find SP-NDK and SP-HAL Dependencies - -VNDK Definition Tool can define the same-process HAL as well. To find SP-NDK, -SP-HAL, SP-HAL-DEP, and VNDK-stable, run `sp-lib` subcommand: - - $ python3 vndk_definition_tool.py sp-lib \ - --system ${ANDROID_PRODUCT_OUT}/system \ - --vendor ${ANDROID_PRODUCT_OUT}/vendor - -The output format is identical to the one described in [Usage](#usage) section. - - -## Python 2 Support - -Since `vndk_definition_tool.py` runs 3x faster with Python 3, the shebang is -specifying `python3` by default. To run `vndk_definition_tool.py` with -python2, run the following command: - - $ python vndk_definition_tool.py [options] + --system system \ + --vendor vendor \ + --aosp-system /path/to/aosp/generic/system \ + --tag-file eligible-list-v3.0.csv diff --git a/vndk/tools/definition-tool/templates/extra_vndk_sp_indirect.txt b/vndk/tools/definition-tool/templates/extra_vndk_sp_indirect.txt new file mode 100644 index 000000000..58e9b5110 --- /dev/null +++ b/vndk/tools/definition-tool/templates/extra_vndk_sp_indirect.txt @@ -0,0 +1,39 @@ +LOCAL_PATH := $(call my-dir) + +EXTRA_VNDK_SP_INDIRECT_LIBRARIES := ##_EXTRA_VNDK_SP_INDIRECT_## + +define define-vndk-sp-indirect-ext-lib +include $$(CLEAR_VARS) +LOCAL_MODULE := $1.vndk-sp-ext +LOCAL_MODULE_CLASS := SHARED_LIBRARIES +LOCAL_PREBUILT_MODULE_FILE := $$(call intermediates-dir-for,SHARED_LIBRARIES,$1,,,,)/PACKED/$1.so +LOCAL_MULTILIB := first +LOCAL_MODULE_TAGS := optional +LOCAL_INSTALLED_MODULE_STEM := $1.so +LOCAL_MODULE_SUFFIX := .so +LOCAL_MODULE_RELATIVE_PATH := vndk-sp +LOCAL_VENDOR_MODULE := true +include $$(BUILD_PREBUILT) + +ifneq ($$(TARGET_2ND_ARCH),) +ifneq ($$(TARGET_TRANSLATE_2ND_ARCH),true) +include $$(CLEAR_VARS) +LOCAL_MODULE := $1.vndk-sp-ext +LOCAL_MODULE_CLASS := SHARED_LIBRARIES +LOCAL_PREBUILT_MODULE_FILE := $$(call intermediates-dir-for,SHARED_LIBRARIES,$1,,,$$(TARGET_2ND_ARCH_VAR_PREFIX),)/PACKED/$1.so +LOCAL_MULTILIB := 32 +LOCAL_MODULE_TAGS := optional +LOCAL_INSTALLED_MODULE_STEM := $1.so +LOCAL_MODULE_SUFFIX := .so +LOCAL_MODULE_RELATIVE_PATH := vndk-sp +LOCAL_VENDOR_MODULE := true +include $$(BUILD_PREBUILT) +endif # TARGET_TRANSLATE_2ND_ARCH is not true +endif # TARGET_2ND_ARCH is not empty +endef + +$(foreach lib,$(VNDK_SP_LIBRARIES),\ + $(eval $(call define-vndk-sp-indirect-ext-lib,$(lib)))) + +# Add following module names to PRODUCT_PACKAGES: +# PRODUCT_PACKAGES += $(addsuffix .vndk-sp-ext,$(EXTRA_VNDK_SP_INDIRECT_LIBRARIES)) diff --git a/vndk/tools/definition-tool/templates/vndk_ext.txt b/vndk/tools/definition-tool/templates/vndk_ext.txt new file mode 100644 index 000000000..307128056 --- /dev/null +++ b/vndk/tools/definition-tool/templates/vndk_ext.txt @@ -0,0 +1,37 @@ +LOCAL_PATH := $(call my-dir) + +VNDK_EXT_LIBRARIES := ##_VNDK_EXT_## + +define define-vndk-ext-lib +include $$(CLEAR_VARS) +LOCAL_MODULE := $1.vndk-ext +LOCAL_MODULE_CLASS := SHARED_LIBRARIES +LOCAL_PREBUILT_MODULE_FILE := $$(call intermediates-dir-for,SHARED_LIBRARIES,$1,,,,)/PACKED/$1.so +LOCAL_MULTILIB := first +LOCAL_MODULE_TAGS := optional +LOCAL_INSTALLED_MODULE_STEM := $1.so +LOCAL_MODULE_SUFFIX := .so +LOCAL_VENDOR_MODULE := true +include $$(BUILD_PREBUILT) + +ifneq ($$(TARGET_2ND_ARCH),) +ifneq ($$(TARGET_TRANSLATE_2ND_ARCH),true) +include $$(CLEAR_VARS) +LOCAL_MODULE := $1.vndk-ext +LOCAL_MODULE_CLASS := SHARED_LIBRARIES +LOCAL_PREBUILT_MODULE_FILE := $$(call intermediates-dir-for,SHARED_LIBRARIES,$1,,,$$(TARGET_2ND_ARCH_VAR_PREFIX),)/PACKED/$1.so +LOCAL_MULTILIB := 32 +LOCAL_MODULE_TAGS := optional +LOCAL_INSTALLED_MODULE_STEM := $1.so +LOCAL_MODULE_SUFFIX := .so +LOCAL_VENDOR_MODULE := true +include $$(BUILD_PREBUILT) +endif # TARGET_TRANSLATE_2ND_ARCH is not true +endif # TARGET_2ND_ARCH is not empty +endef + +$(foreach lib,$(VNDK_SP_LIBRARIES),\ + $(eval $(call define-vndk-ext-lib,$(lib)))) + +# Add following module names to PRODUCT_PACKAGES: +# PRODUCT_PACKAGES += $(addsuffix .vndk-ext,$(VNDK_EXT_LIBRARIES)) From 47f5097cf813d33613be083babf804dc071dfbe9 Mon Sep 17 00:00:00 2001 From: Logan Chien Date: Thu, 8 Jun 2017 01:55:25 +0800 Subject: [PATCH 4/4] vndk-def: Add --system-dir-ignored option This commit adds --system-dir-ignored option so that the user can ignore /system/vendor directory and run the following command on some devices: python3 vndk_definition_tool.py vndk \ --system ${ANDROID_PRODUCT_OUT}/system \ --system-dir-ignored vendor \ --vendor ${ANDROID_PRODUCT_OUT}/vendor Test: ./tests/run.py Test: Add --system-dir-ignored to make sure /system/vendor on some devices are not treated as system modules. Change-Id: I45a82bf81f2e2a6b34c146310f53d47dfe2313c3 --- vndk/tools/definition-tool/README.md | 13 +++ .../definition-tool/vndk_definition_tool.py | 92 ++++++++++--------- 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/vndk/tools/definition-tool/README.md b/vndk/tools/definition-tool/README.md index 37363fc0a..acf582464 100644 --- a/vndk/tools/definition-tool/README.md +++ b/vndk/tools/definition-tool/README.md @@ -61,6 +61,19 @@ their `device.mk`. For example, in the example mentioned above, following PRODUCT_PACKAGES += libexample4.vndk-ext +## Ignore Subdirectories + +Some devices keep their vendor modules in `/system/vendor`. To run VNDK +definition tool for those devices, we have to skip `/system/vendor` and specify +it with `--vendor` option. For example: + + python3 vndk_definition_tool.py vndk \ + --system ${ANDROID_PRODUCT_OUT}/system \ + --system-dir-igored vendor \ + --vendor ${ANDROID_PRODUCT_OUT}/system/vendor \ + # ... + + ## Implicit Dependencies If there are implicit dependencies, such as `dlopen()`, we can specify them in diff --git a/vndk/tools/definition-tool/vndk_definition_tool.py b/vndk/tools/definition-tool/vndk_definition_tool.py index 46c587f0b..aeefd2cbc 100755 --- a/vndk/tools/definition-tool/vndk_definition_tool.py +++ b/vndk/tools/definition-tool/vndk_definition_tool.py @@ -948,15 +948,20 @@ class ELFLinker(object): return re.compile('|'.join(patts)) def add_executables_in_dir(self, partition_name, partition, root, - alter_partition, alter_subdirs, scan_elf_files): + alter_partition, alter_subdirs, ignored_subdirs, + scan_elf_files): root = os.path.abspath(root) prefix_len = len(root) + 1 if alter_subdirs: alter_patt = ELFLinker._compile_path_matcher(root, alter_subdirs) + if ignored_subdirs: + ignored_patt = ELFLinker._compile_path_matcher(root, ignored_subdirs) for path, elf in scan_elf_files(root): short_path = os.path.join('/', partition_name, path[prefix_len:]) + if ignored_subdirs and ignored_patt.match(path): + continue if alter_subdirs and alter_patt.match(path): self.add_lib(alter_partition, short_path, elf) else: @@ -1488,20 +1493,23 @@ class ELFLinker(object): @staticmethod def _create_internal(scan_elf_files, system_dirs, system_dirs_as_vendor, - vendor_dirs, vendor_dirs_as_system, extra_deps, - generic_refs): + system_dirs_ignored, vendor_dirs, + vendor_dirs_as_system, vendor_dirs_ignored, + extra_deps, generic_refs): graph = ELFLinker() if system_dirs: for path in system_dirs: graph.add_executables_in_dir('system', PT_SYSTEM, path, PT_VENDOR, system_dirs_as_vendor, + system_dirs_ignored, scan_elf_files) if vendor_dirs: for path in vendor_dirs: graph.add_executables_in_dir('vendor', PT_VENDOR, path, PT_SYSTEM, vendor_dirs_as_system, + vendor_dirs_ignored, scan_elf_files) if extra_deps: @@ -1513,11 +1521,14 @@ class ELFLinker(object): return graph @staticmethod - def create(system_dirs=None, system_dirs_as_vendor=None, vendor_dirs=None, - vendor_dirs_as_system=None, extra_deps=None, generic_refs=None): + def create(system_dirs=None, system_dirs_as_vendor=None, + system_dirs_ignored=None, vendor_dirs=None, + vendor_dirs_as_system=None, vendor_dirs_ignored=None, + extra_deps=None, generic_refs=None): return ELFLinker._create_internal( - scan_elf_files, system_dirs, system_dirs_as_vendor, vendor_dirs, - vendor_dirs_as_system, extra_deps, generic_refs) + scan_elf_files, system_dirs, system_dirs_as_vendor, + system_dirs_ignored, vendor_dirs, vendor_dirs_as_system, + vendor_dirs_ignored, extra_deps, generic_refs) @staticmethod def create_from_dump(system_dirs=None, system_dirs_as_vendor=None, @@ -1672,10 +1683,18 @@ class ELFGraphCommand(Command): '--system-dir-as-vendor', action='append', help='sub directory of system partition that has vendor files') + parser.add_argument( + '--system-dir-ignored', action='append', + help='sub directory of system partition that must be ignored') + parser.add_argument( '--vendor-dir-as-system', action='append', help='sub directory of vendor partition that has system files') + parser.add_argument( + '--vendor-dir-ignored', action='append', + help='sub directory of vendor partition that must be ignored') + parser.add_argument( '--load-generic-refs', help='compare with generic reference symbols') @@ -1692,14 +1711,6 @@ class ELFGraphCommand(Command): '/system') return None - -class VNDKCommandBase(ELFGraphCommand): - def add_argparser_options(self, parser): - super(VNDKCommandBase, self).add_argparser_options(parser) - - parser.add_argument('--no-default-dlopen-deps', action='store_true', - help='do not add default dlopen dependencies') - def _check_arg_dir_exists(self, arg_name, dirs): for path in dirs: if not os.path.exists(path): @@ -1716,17 +1727,33 @@ class VNDKCommandBase(ELFGraphCommand): self._check_arg_dir_exists('--vendor', args.vendor) def create_from_args(self, args): - """Create all essential data structures for VNDK computation.""" - self.check_dirs_from_args(args) generic_refs = self.get_generic_refs_from_args(args) graph = ELFLinker.create(args.system, args.system_dir_as_vendor, + args.system_dir_ignored, args.vendor, args.vendor_dir_as_system, + args.vendor_dir_ignored, args.load_extra_deps, generic_refs=generic_refs) + return (generic_refs, graph) + + +class VNDKCommandBase(ELFGraphCommand): + def add_argparser_options(self, parser): + super(VNDKCommandBase, self).add_argparser_options(parser) + + parser.add_argument('--no-default-dlopen-deps', action='store_true', + help='do not add default dlopen dependencies') + + def create_from_args(self, args): + """Create all essential data structures for VNDK computation.""" + + generic_refs, graph = \ + super(VNDKCommandBase, self).create_from_args(args) + if not args.no_default_dlopen_deps: script_dir = os.path.dirname(os.path.abspath(__file__)) minimum_dlopen_deps = os.path.join(script_dir, 'datasets', @@ -1921,9 +1948,7 @@ class VNDKCapCommand(ELFGraphCommand): super(VNDKCapCommand, self).add_argparser_options(parser) def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) banned_libs = BannedLibDict.create_default() @@ -1954,9 +1979,7 @@ class DepsCommand(ELFGraphCommand): help='print symbols') def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) results = [] for partition in range(NUM_PARTITIONS): @@ -2012,9 +2035,7 @@ class DepsClosureCommand(ELFGraphCommand): help='exclude ndk libraries') def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) # Find root/excluded libraries by their paths. def report_error(path): @@ -2299,9 +2320,7 @@ class CheckDepCommand(ELFGraphCommand): return num_errors def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) tagged_paths = TaggedPathDict.create_from_csv_path(args.tag_file) tagged_libs = TaggedLibDict.create_from_graph(graph, tagged_paths) @@ -2378,9 +2397,7 @@ class DepGraphCommand(ELFGraphCommand): return data, violate_libs def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) tagged_paths = TaggedPathDict.create_from_csv_path(args.tag_file) data, violate_libs = self._get_dep_graph(graph, tagged_paths) @@ -2410,9 +2427,7 @@ class VNDKSPCommand(ELFGraphCommand): super(VNDKSPCommand, self).add_argparser_options(parser) def main(self, args): - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) + generic_refs, graph = self.create_from_args(args) vndk_sp = graph.compute_predefined_vndk_sp() for lib in sorted_lib_path_list(vndk_sp): @@ -2432,12 +2447,7 @@ class SpLibCommand(ELFGraphCommand): super(SpLibCommand, self).add_argparser_options(parser) def main(self, args): - generic_refs = self.get_generic_refs_from_args(args) - - graph = ELFLinker.create(args.system, args.system_dir_as_vendor, - args.vendor, args.vendor_dir_as_system, - args.load_extra_deps) - + generic_refs, graph = self.create_from_args(args) print_sp_lib(graph.compute_sp_lib(generic_refs)) return 0