gsi_util: adding dump subcommand

'dump' command can dump information from given image, which could
be a image file, folder or device by adb.

Use $./gsi_util.py dump --help for the detail.

The patch also includes a "dump" framework, to implement some
dumpers to dump information. This patch also includes PropDumper to
dump information from property files, and XmlDumper to dump
information from XML files.

There is an initial dump info list in dump_info_list.py.
Use subcommand 'list_dump' could output the list.

Usually using Dumper is enough to dump information. dump.py is an
example to use Dumper.

Bug: 70253764
Test: dump from different mounter
Change-Id: I7c05f8f24d44d3c7429f2c428963f64191f49a53

gsi/gsi_util/Android.bp

@@ -18,6 +18,7 @@ python_binary_host {
     "gsi_util.py",
     "gsi_util/*.py",
     "gsi_util/commands/*.py",
+    "gsi_util/dumpers/*.py",
     "gsi_util/mounters/*.py",
     "gsi_util/utils/*.py",
   ],

gsi/gsi_util/gsi_util.py

@@ -28,7 +28,7 @@ class GsiUtil(object):
 
   # Adds gsi_util COMMAND here.
   # TODO(bowgotsai): auto collect from gsi_util/commands/*.py
-  _COMMANDS = ['flash_gsi', 'pull', 'hello']
+  _COMMANDS = ['flash_gsi', 'pull', 'dump', 'hello']
 
   _LOGGING_FORMAT = '%(message)s'
   _LOGGING_LEVEL = logging.WARNING

gsi/gsi_util/gsi_util/commands/dump.py

@@ -0,0 +1,168 @@
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gsi_util command 'dump'."""
+
+import argparse
+import logging
+import sys
+
+from gsi_util.dumpers.dumper import Dumper
+from gsi_util.mounters.composite_mounter import CompositeMounter
+
+
+class DumpReporter(object):
+  """Format and output dump info result to a output stream.
+
+  When constructing DumpReporter, you need to give os and name_list.
+  os is the stream to output the formatted, which should be inherited from
+  io.IOBase. name_list is a string list describe the info names to be output.
+
+  After collected all dump result, calls output() to output the
+  dump_result_dict. dump_result_dict is a dictionary that maps info names to
+  theirs result values.
+  """
+
+  _UNKNOWN_VALUE = '<unknown>'
+
+  def __init__(self, os, name_list):
+    """Inits DumpReporter with an output stream and an info name list.
+
+    Args:
+      os: the output stream of outputing the report
+      name_list: the info name list will be output
+    """
+    self._os = os
+    self._name_list = name_list
+    self._show_unknown = False
+
+  def set_show_unknown(self):
+    """Enable force output dump info without dump result.
+
+    By default, it doesn't output the dump info in the info name list which
+    is not in dump result, i.e. the dump_result_dict of output().
+    """
+    self._show_unknown = True
+
+  def _output_dump_info(self, info_name, value):
+    print >> self._os, '{:30}: {}'.format(info_name, value)
+
+  def output(self, dump_result_dict):
+    """Output the given dump result.
+
+    Args:
+      dump_result_dict: the dump result dictionary to be output
+    """
+    for info_name in self._name_list:
+      value = dump_result_dict.get(info_name)
+      if not value:
+        if not self._show_unknown:
+          continue
+        value = self._UNKNOWN_VALUE
+
+      self._output_dump_info(info_name, value)
+
+
+def do_list_dump(_):
+  for info in Dumper.get_all_dump_list():
+    print info.info_name
+
+
+def do_dump(args):
+  logging.info('==== DUMP ====')
+  logging.info('  system=%s vendor=%s', args.system, args.vendor)
+  if not args.system and not args.vendor:
+    sys.exit('Without system nor vendor.')
+
+  mounter = CompositeMounter()
+  if args.system:
+    mounter.add_by_mount_target('system', args.system)
+  if args.vendor:
+    mounter.add_by_mount_target('vendor', args.vendor)
+
+  logging.debug('Info name list: %s', args.INFO_NAME)
+  dump_list = Dumper.make_dump_list_by_name_list(args.INFO_NAME) if len(
+      args.INFO_NAME) else Dumper.get_all_dump_list()
+
+  with mounter as file_accessor:
+    dumper = Dumper(file_accessor)
+    dump_result_dict = dumper.dump(dump_list)
+
+  # reserved for output to a file
+  os = sys.stdout
+  reporter = DumpReporter(os, (x.info_name for x in dump_list))
+  if args.show_unknown:
+    reporter.set_show_unknown()
+  reporter.output(dump_result_dict)
+
+  logging.info('==== DONE ====')
+
+
+DUMP_DESCRIPTION = """'dump' command dumps information from given image
+
+You must assign at least one image source by SYSTEM and/or VENDOR.
+Image source could be:
+
+ adb[:SERIAL_NUM]: form the device which be connected with adb
+  image file name: from the given image file, e.g. the file name of a GSI.
+                   If a image file is assigned to be the source of system
+                   image, gsu_util will detect system-as-root automatically.
+      folder name: from the given folder, e.g. the system/vendor folder in an
+                   Android build out folder.
+
+You could use command 'list_dump' to query all info names:
+
+    $ ./gsi_util.py list_dump
+
+For example you could use following command to query the security patch level
+in an system image file:
+
+    $ ./gsi_util.py dump --system system.img system_security_patch_level
+
+You there is no given INFO_NAME, all information will be dumped.
+
+Here are some other usage examples:
+
+    $ ./gsi_util.py dump --system adb --vendor adb
+    $ ./gsi_util.py dump --system system.img --show-unknown
+    $ ./gsi_util.py dump --system my/out/folder/system"""
+
+
+def setup_command_args(parser):
+  # command 'list_dump'
+  list_dump_parser = parser.add_parser(
+      'list_dump', help='list all possible info names')
+  list_dump_parser.set_defaults(func=do_list_dump)
+
+  # command 'dump'
+  dump_parser = parser.add_parser(
+      'dump',
+      help='dump information from given image',
+      description=DUMP_DESCRIPTION,
+      formatter_class=argparse.RawTextHelpFormatter)
+  dump_parser.add_argument(
+      '--system', type=str, help='system image file name, folder name or "adb"')
+  dump_parser.add_argument(
+      '--vendor', type=str, help='vendor image file name, folder name or "adb"')
+  dump_parser.add_argument(
+      '-u',
+      '--show-unknown',
+      action='store_true',
+      help='force display the dump info items in list which does not exist')
+  dump_parser.add_argument(
+      'INFO_NAME',
+      type=str,
+      nargs='*',
+      help='the info name to be dumped. Dump all if not given')
+  dump_parser.set_defaults(func=do_dump)

gsi/gsi_util/gsi_util/dumpers/dump_info_list.py

@@ -0,0 +1,44 @@
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provide the information list for command 'dump'."""
+
+from collections import namedtuple
+
+from gsi_util.dumpers.prop_dumper import PropDumper
+from gsi_util.dumpers.xml_dumper import XmlDumper
+
+SYSTEM_MATRIX_DUMPER = (XmlDumper, '/system/compatibility_matrix.xml')
+SYSTEM_BUILD_PROP_DUMPER = (PropDumper, '/system/build.prop')
+SYSTEM_MANIFEST_DUMPER = (PropDumper, '/system/manifest.xml')
+
+VENDOR_DEFAULT_PROP_DUMPER = (PropDumper, '/vendor/default.prop')
+VENDOR_BUILD_PROP_DUMPER = (PropDumper, '/vendor/build.prop')
+
+DumpInfoListItem = namedtuple('DumpInfoListItem',
+                              'info_name dumper_create_args lookup_key')
+
+# The total list of all possible dump info.
+# It will be output by the order of the list.
+DUMP_LIST = [
+    DumpInfoListItem('system_build_id', SYSTEM_BUILD_PROP_DUMPER, 'ro.build.display.id'),
+    DumpInfoListItem('system_sdk_ver', SYSTEM_BUILD_PROP_DUMPER, 'ro.build.version.sdk'),
+    DumpInfoListItem('system_security_patch_level', SYSTEM_BUILD_PROP_DUMPER, 'ro.build.version.security_patch'),
+    DumpInfoListItem('system_kernel_sepolicy_ver', SYSTEM_MATRIX_DUMPER, './sepolicy/kernel-sepolicy-version'),
+    DumpInfoListItem('system_support_sepolicy_ver', SYSTEM_MATRIX_DUMPER, './sepolicy/sepolicy-version'),
+    DumpInfoListItem('system_avb_ver', SYSTEM_MATRIX_DUMPER, './avb/vbmeta-version'),
+    DumpInfoListItem('vendor_fingerprint', VENDOR_BUILD_PROP_DUMPER, 'ro.vendor.build.fingerprint'),
+    DumpInfoListItem('vendor_low_ram', VENDOR_BUILD_PROP_DUMPER, 'ro.config.low_ram'),
+    DumpInfoListItem('vendor_zygote', VENDOR_DEFAULT_PROP_DUMPER, 'ro.zygote'),
+]  # pyformat: disable

gsi/gsi_util/gsi_util/dumpers/dumper.py

@@ -0,0 +1,86 @@
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implement dump methods and utils to dump info from a mounter."""
+
+from gsi_util.dumpers.dump_info_list import DUMP_LIST
+
+
+class Dumper(object):
+
+  def __init__(self, file_accessor):
+    self._file_accessor = file_accessor
+
+  @staticmethod
+  def _dump_by_dumper(dumper_instance, dump_list):
+    """Dump info by the given dumper instance according the given dump_list.
+
+    Used for dump(), see the comment of dump() for type details.
+
+    Args:
+      dumper_instance: a dumper instance to process dump.
+      dump_list: a list of dump info to be dump. The items in the list must
+        relative to dumper_instance.
+
+    Returns:
+      The dump result by dictionary maps info_name to the value of dump result.
+    """
+    dump_result = {}
+
+    for dump_info in dump_list:
+      value = dumper_instance.dump(dump_info.lookup_key)
+      dump_result[dump_info.info_name] = value
+
+    return dump_result
+
+  def dump(self, dump_list):
+    """Dump info according the given dump_list.
+
+    Args:
+      dump_list: a list of dump info to be dump. See dump_info_list.py for
+                 the detail types.
+    Returns:
+      The dump result by dictionary maps info_name to the value of dump result.
+    """
+    dump_result = {}
+
+    # query how many different dumpers to dump
+    dumper_set = set([x.dumper_create_args for x in dump_list])
+    for dumper_create_args in dumper_set:
+      # The type of a dumper_create_args is (Class, instantiation args...)
+      dumper_class = dumper_create_args[0]
+      dumper_args = dumper_create_args[1:]
+      # Create the dumper
+      with dumper_class(self._file_accessor, dumper_args) as dumper_instance:
+        dump_list_for_the_dumper = (
+            x for x in dump_list if x.dumper_create_args == dumper_create_args)
+        dumper_result = self._dump_by_dumper(dumper_instance,
+                                             dump_list_for_the_dumper)
+        dump_result.update(dumper_result)
+
+    return dump_result
+
+  @staticmethod
+  def make_dump_list_by_name_list(name_list):
+    info_list = []
+    for info_name in name_list:
+      info = next((x for x in DUMP_LIST if x.info_name == info_name), None)
+      if not info:
+        raise RuntimeError('Unknown info name: "{}"'.format(info_name))
+      info_list.append(info)
+    return info_list
+
+  @staticmethod
+  def get_all_dump_list():
+    return DUMP_LIST

gsi/gsi_util/gsi_util/dumpers/prop_dumper.py

@@ -0,0 +1,44 @@
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides class PropDumper."""
+
+import logging
+import re
+
+
+class PropDumper(object):
+
+  def __init__(self, file_accessor, args):
+    filename_in_mount = args[0]
+    logging.debug('Parse %s...', filename_in_mount)
+    with file_accessor.prepare_file(filename_in_mount) as filename:
+      if filename:
+        with open(filename) as fp:
+          self._content = fp.read()
+
+  def __enter__(self):
+    # do nothing
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    if hasattr(self, '_content'):
+      del self._content
+
+  def dump(self, lookup_key):
+    if not hasattr(self, '_content'):
+      return None
+
+    match = re.search('%s=(.*)' % (lookup_key), self._content)
+    return match.group(1) if match else None

gsi/gsi_util/gsi_util/dumpers/xml_dumper.py

@@ -0,0 +1,44 @@
+# Copyright 2017 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides class XmlDumper."""
+
+import logging
+import xml.etree.ElementTree as ET
+
+
+class XmlDumper(object):
+
+  def __init__(self, file_accessor, args):
+    filename_in_mount = args[0]
+    logging.debug('Parse %s...', filename_in_mount)
+    with file_accessor.prepare_file(filename_in_mount) as filename:
+      if filename:
+        self._tree = ET.parse(filename)
+
+  def __enter__(self):
+    # do nothing
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    if hasattr(self, '_tree'):
+      del self._tree
+
+  def dump(self, lookup_key):
+    if not hasattr(self, '_tree'):
+      return None
+
+    xpath = lookup_key
+    results = self._tree.findall(xpath)
+    return ', '.join([e.text for e in results]) if results else None