Refactor native heap viewer to add tests.

This only includes a few tests to start with.

Bug: 62492960

Test: Ran new unit tests.
Test: Ran with all different options and verified it produces the same
Test: output as the previous script.
Change-Id: Iad29a5f04f49986139c92030a3259cae512859af

scripts/native_heapdump_viewer.py

@@ -22,7 +22,8 @@ import re
 import subprocess
 import sys
 
-usage = """
+class Args:
+  _usage = """
 Usage:
 1. Collect a native heap dump from the device. For example:
    $ adb shell stop
@@ -50,40 +51,39 @@ Usage:
    stack frame. 71b07bc0b0 is the address of the stack frame.
 """
 
-verbose = False
+  def __init__(self):
-html_output = False
+    self.verbose = False
-reverse_frames = False
+    self.html_output = False
-product_out = os.getenv("ANDROID_PRODUCT_OUT")
+    self.reverse_frames = False
-if product_out:
+    product_out = os.getenv("ANDROID_PRODUCT_OUT")
-    symboldir = product_out + "/symbols"
+    if product_out:
-else:
+      self.symboldir = product_out + "/symbols"
-    symboldir = "./symbols"
-
-args = sys.argv[1:]
-while len(args) > 1:
-    if args[0] == "--symbols":
-        symboldir = args[1]
-        args = args[2:]
-    elif args[0] == "--verbose":
-        verbose = True
-        args = args[1:]
-    elif args[0] == "--html":
-        html_output = True
-        args = args[1:]
-    elif args[0] == "--reverse":
-        reverse_frames = True
-        args = args[1:]
     else:
-        print "Invalid option "+args[0]
+      self.symboldir = "./symbols"
-        break
 
-if len(args) != 1:
+    i = 1
-    print usage
+    extra_args = []
-    exit(0)
+    while i < len(sys.argv):
+      if sys.argv[i] == "--symbols":
+        i += 1
+        self.symboldir = args[i]
+      elif sys.argv[i] == "--verbose":
+        self.verbose = True
+      elif sys.argv[i] == "--html":
+        self.html_output = True
+      elif sys.argv[i] == "--reverse":
+        self.reverse_frames = True
+      elif sys.argv[i][0] == '-':
+        print "Invalid option " + sys.argv[i]
+      else:
+        extra_args.append(sys.argv[i])
+      i += 1
 
-native_heap = args[0]
+    if len(extra_args) != 1:
+      print self._usage
+      sys.exit(1)
 
-re_map = re.compile("(?P<start>[0-9a-f]+)-(?P<end>[0-9a-f]+) .... (?P<offset>[0-9a-f]+) [0-9a-f]+:[0-9a-f]+ [0-9]+ +(?P<name>.*)")
+    self.native_heap = extra_args[0]
 
 class Backtrace:
   def __init__(self, is_zygote, size, num_allocs, frames):
@@ -105,7 +105,6 @@ class FrameDescription:
     self.location = location
     self.library = library
 
-
 def GetVersion(native_heap):
   """Get the version of the native heap dump."""
 
@@ -117,8 +116,8 @@ def GetVersion(native_heap):
       return m.group('version')
   return None
 
-def NumFieldValid(native_heap):
+def GetNumFieldValidByParsingLines(native_heap):
-  """Determine if the num field is valid.
+  """Determine if the num field is valid by parsing the backtrace lines.
 
   Malloc debug for N incorrectly set the num field to the number of
   backtraces instead of the number of allocations with the same size and
@@ -156,20 +155,30 @@ def NumFieldValid(native_heap):
           return True
   return matched == 0
 
-version = GetVersion(native_heap)
+def GetNumFieldValid(native_heap):
-if not version or version == "v1.0":
+  version = GetVersion(native_heap)
+  if not version or version == "v1.0":
     # Version v1.0 was produced by a buggy version of malloc debug where the
     # num field was set incorrectly.
     # Unfortunately, Android P produced a v1.0 version that does set the
     # num field. Do one more check to see if this is the broken version.
-  num_field_valid = NumFieldValid(native_heap)
+    return GetNumFieldValidByParsingLines(native_heap)
-else:
+  else:
-  num_field_valid = True
+    return True
 
-backtraces = []
+def ParseNativeHeap(native_heap, reverse_frames, num_field_valid):
-mappings = []
+  """Parse the native heap into backtraces, maps.
 
-for line in open(native_heap, "r"):
+  Returns two lists, the first is a list of all of the backtraces, the
+  second is the sorted list of maps.
+  """
+
+  backtraces = []
+  mappings = []
+
+  re_map = re.compile("(?P<start>[0-9a-f]+)-(?P<end>[0-9a-f]+) .... (?P<offset>[0-9a-f]+) [0-9a-f]+:[0-9a-f]+ [0-9]+ +(?P<name>.*)")
+
+  for line in open(native_heap, "r"):
     # Format of line:
     #   z 0  sz       50  num    1  bt 000000000000a100 000000000000b200
     parts = line.split()
@@ -184,8 +193,9 @@ for line in open(native_heap, "r"):
       if reverse_frames:
         frames = list(reversed(frames))
       backtraces.append(Backtrace(is_zygote, size, num_allocs, frames))
-        continue
+    else:
-
+      # Parse map line:
+      #   720de01000-720ded7000 r-xp 00000000 fd:00 495  /system/lib64/libc.so
       m = re_map.match(line)
       if m:
         start = int(m.group('start'), 16)
@@ -193,11 +203,16 @@ for line in open(native_heap, "r"):
         offset = int(m.group('offset'), 16)
         name = m.group('name')
         mappings.append(Mapping(start, end, offset, name))
-        continue
 
-# Return the mapping that contains the given address.
+  return backtraces, mappings
-# Returns None if there is no such mapping.
+
-def find_mapping(addr):
+def FindMapping(mappings, addr):
+  """Find the mapping given addr.
+
+  Returns the mapping that contains addr.
+  Returns None if there is no such mapping.
+  """
+
   min = 0
   max = len(mappings) - 1
   while True:
@@ -211,32 +226,39 @@ def find_mapping(addr):
     else:
       return mappings[mid]
 
-# Resolve address libraries and offsets.
+
-# addr_offsets maps addr to .so file offset
+def ResolveAddrs(html_output, symboldir, backtraces, mappings):
-# addrs_by_lib maps library to list of addrs from that library
+  """Resolve address libraries and offsets.
-# Resolved addrs maps addr to FrameDescription
+
-addr_offsets = {}
+  addr_offsets maps addr to .so file offset
-addrs_by_lib = {}
+  addrs_by_lib maps library to list of addrs from that library
-resolved_addrs = {}
+  Resolved addrs maps addr to FrameDescription
-EMPTY_FRAME_DESCRIPTION = FrameDescription("???", "???", "???")
+
-for backtrace in backtraces:
+  Returns the resolved_addrs hash.
+  """
+
+  addr_offsets = {}
+  addrs_by_lib = {}
+  resolved_addrs = {}
+  empty_frame_description = FrameDescription("???", "???", "???")
+  for backtrace in backtraces:
     for addr in backtrace.frames:
       if addr in addr_offsets:
         continue
-        mapping = find_mapping(addr)
+      mapping = FindMapping(mappings, addr)
       if mapping:
         addr_offsets[addr] = addr - mapping.start + mapping.offset
         if not (mapping.name in addrs_by_lib):
           addrs_by_lib[mapping.name] = []
         addrs_by_lib[mapping.name].append(addr)
       else:
-            resolved_addrs[addr] = EMPTY_FRAME_DESCRIPTION
+        resolved_addrs[addr] = empty_frame_description
 
-
+  # Resolve functions and line numbers.
-# Resolve functions and line numbers
+  if html_output == False:
-if html_output == False:
     print "Resolving symbols using directory %s..." % symboldir
-for lib in addrs_by_lib:
+
+  for lib in addrs_by_lib:
     sofile = symboldir + lib
     if os.path.isfile(sofile):
       file_offset = 0
@@ -250,6 +272,7 @@ for lib in addrs_by_lib:
       input_addrs = ""
       for addr in addrs_by_lib[lib]:
         input_addrs += "%s\n" % hex(addr_offsets[addr] - file_offset)
+
       p = subprocess.Popen(["addr2line", "-C", "-j", ".text", "-e", sofile, "-f"], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
       result = p.communicate(input_addrs)[0]
       splitted = result.split("\n")
@@ -257,15 +280,17 @@ for lib in addrs_by_lib:
         function = splitted[2*x];
         location = splitted[2*x+1];
         resolved_addrs[addrs_by_lib[lib][x]] = FrameDescription(function, location, lib)
-
     else:
       if html_output == False:
         print "%s not found for symbol resolution" % lib
+
       fd = FrameDescription("???", "???", lib)
       for addr in addrs_by_lib[lib]:
         resolved_addrs[addr] = fd
 
-def addr2line(addr):
+  return resolved_addrs
+
+def Addr2Line(resolved_addrs, addr):
   if addr == "ZYGOTE" or addr == "APP":
     return FrameDescription("", "", "")
 
@@ -288,11 +313,8 @@ class AddrInfo:
         self.children[child.addr] = child
       self.children[child.addr].addStack(size, num_allocs, stack[1:])
 
-zygote = AddrInfo("ZYGOTE")
+def Display(resolved_addrs, indent, total, parent_total, node):
-app = AddrInfo("APP")
+  fd = Addr2Line(resolved_addrs, node.addr)
-
-def display(indent, total, parent_total, node):
-    fd = addr2line(node.addr)
   total_percent = 0
   if total != 0:
     total_percent = 100 * node.size / float(total)
@@ -302,12 +324,10 @@ def display(indent, total, parent_total, node):
   print "%9d %6.2f%% %6.2f%% %8d %s%s %s %s %s" % (node.size, total_percent, parent_percent, node.number, indent, node.addr, fd.library, fd.function, fd.location)
   children = sorted(node.children.values(), key=lambda x: x.size, reverse=True)
   for child in children:
-        display(indent + "  ", total, node.size, child)
+    Display(resolved_addrs, indent + "  ", total, node.size, child)
 
-label_count=0
+def DisplayHtml(verbose, resolved_addrs, total, node, extra, label_count):
-def display_html(total, node, extra):
+  fd = Addr2Line(resolved_addrs, node.addr)
-    global label_count
-    fd = addr2line(node.addr)
   if verbose:
     lib = fd.library
   else:
@@ -327,24 +347,18 @@ def display_html(total, node, extra):
     print '<label for="' + str(label_count) + '">' + label + '</label>'
     print '<input type="checkbox" id="' + str(label_count) + '"/>'
     print '<ol>'
-        label_count+=1
+    label_count += 1
     for child in children:
-            display_html(total, child, "")
+      label_count = DisplayHtml(verbose, resolved_addrs, total, child, "", label_count)
     print '</ol>'
   else:
     print label
   print '</li>'
-for backtrace in backtraces:
-    stack = []
-    for addr in backtrace.frames:
-        stack.append(AddrInfo("%x" % addr))
-    stack.reverse()
-    if backtrace.is_zygote:
-        zygote.addStack(backtrace.size, backtrace.num_allocs, stack)
-    else:
-        app.addStack(backtrace.size, backtrace.num_allocs, stack)
 
-html_header = """
+  return label_count
+
+def CreateHtml(verbose, app, zygote, resolved_addrs):
+  print """
 <!DOCTYPE html>
 <html><head><style>
 li input {
@@ -367,19 +381,44 @@ label {
 Click on an individual line to expand/collapse to see the details of the
 allocation data<ol>
 """
-html_footer = "</ol></body></html>"
 
-if html_output:
+  label_count = 0
-    print html_header
+  label_count = DisplayHtml(verbose, resolved_addrs, app.size, app, "app ", label_count)
-    display_html(app.size, app, "app ")
+  if zygote.size > 0:
-    if zygote.size>0:
+    DisplayHtml(verbose, resolved_addrs, zygote.size, zygote, "zygote ", label_count)
-        display_html(zygote.size, zygote, "zygote ")
+  print "</ol></body></html>"
-    print html_footer
+
-else:
+def main():
+  args = Args()
+
+  num_field_valid = GetNumFieldValid(args.native_heap)
+
+  backtraces, mappings = ParseNativeHeap(args.native_heap, args.reverse_frames, num_field_valid)
+  # Resolve functions and line numbers
+  resolved_addrs = ResolveAddrs(args.html_output, args.symboldir, backtraces, mappings)
+
+  app = AddrInfo("APP")
+  zygote = AddrInfo("ZYGOTE")
+
+  for backtrace in backtraces:
+    stack = []
+    for addr in backtrace.frames:
+      stack.append(AddrInfo("%x" % addr))
+    stack.reverse()
+    if backtrace.is_zygote:
+      zygote.addStack(backtrace.size, backtrace.num_allocs, stack)
+    else:
+      app.addStack(backtrace.size, backtrace.num_allocs, stack)
+
+  if args.html_output:
+    CreateHtml(args.verbose, app, zygote, resolved_addrs)
+  else:
     print ""
     print "%9s %6s %6s %8s    %s %s %s %s" % ("BYTES", "%TOTAL", "%PARENT", "COUNT", "ADDR", "LIBRARY", "FUNCTION", "LOCATION")
-    display("", app.size, app.size + zygote.size, app)
+    Display(resolved_addrs, "", app.size, app.size + zygote.size, app)
     print ""
-    display("", zygote.size, app.size + zygote.size, zygote)
+    Display(resolved_addrs, "", zygote.size, app.size + zygote.size, zygote)
     print ""
 
+if __name__ == '__main__':
+  main()

scripts/native_heapdump_viewer_tests.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the native_heapdump_viewer script."""
+
+import native_heapdump_viewer
+import os
+import sys
+import tempfile
+import unittest
+
+class NativeHeapdumpViewerTest(unittest.TestCase):
+  _tmp_file_name = None
+
+  def CreateTmpFile(self, contents):
+    fd, self._tmp_file_name = tempfile.mkstemp()
+    os.write(fd, contents)
+    os.close(fd)
+    return self._tmp_file_name
+
+  def tearDown(self):
+    if self._tmp_file_name:
+      try:
+        os.unlink(self._tmp_file_name)
+      except Exception:
+        print "Failed to delete " + heap
+
+class GetNumFieldValidTest(NativeHeapdumpViewerTest):
+  _map_data = """
+MAPS
+1000-10000 r-xp 00000000 fd:00 495                            /data/does_not_exist.so
+END
+"""
+
+  _heap_num_field_valid_version10 = """
+Android Native Heap Dump v1.0
+
+Total memory: 33800
+Allocation records: 13
+Backtrace size: 16
+
+z 1  sz   1000  num    4  bt 1000 2000 3000
+z 1  sz   2000  num    6  bt 1100 2100 3100
+z 0  sz   1200  num    1  bt 1200 2200 3200
+z 0  sz   8300  num    2  bt 1300 2300 3300
+"""
+
+  _heap_num_field_invalid_version10 = """
+Android Native Heap Dump v1.0
+
+Total memory: 12500
+Allocation records: 4
+Backtrace size: 16
+
+z 1  sz   1000  num    16  bt 1000 2000 3000
+z 1  sz   2000  num    16  bt 1100 2100 3100
+z 0  sz   1200  num    16  bt 1200 2200 3200
+z 0  sz   8300  num    16  bt 1300 2300 3300
+"""
+
+  _heap_data = """
+
+Total memory: 200000
+Allocation records: 64
+Backtrace size: 16
+
+z 1  sz   1000  num    16  bt 1000 2000 3000
+z 1  sz   2000  num    16  bt 1100 2100 3100
+z 0  sz   1200  num    16  bt 1200 2200 3200
+z 0  sz   8300  num    16  bt 1300 2300 3300
+"""
+
+  def test_version10_valid(self):
+    heap = self.CreateTmpFile(self._heap_num_field_valid_version10 + self._map_data)
+    self.assertTrue(native_heapdump_viewer.GetNumFieldValid(heap))
+
+  def test_version10_invalid(self):
+    heap = self.CreateTmpFile(self._heap_num_field_invalid_version10 + self._map_data)
+    self.assertFalse(native_heapdump_viewer.GetNumFieldValid(heap))
+
+  def test_version11_valid(self):
+    heap = self.CreateTmpFile("Android Native Heap Dump v1.1" + self._heap_data + self._map_data)
+    self.assertTrue(native_heapdump_viewer.GetNumFieldValid(heap))
+
+  def test_version12_valid(self):
+    heap = self.CreateTmpFile("Android Native Heap Dump v1.2" + self._heap_data + self._map_data)
+    self.assertTrue(native_heapdump_viewer.GetNumFieldValid(heap))
+
+class ParseNativeHeapTest(NativeHeapdumpViewerTest):
+  _backtrace_data = """
+z 1  sz   1000  num    4  bt 1000 2000 3000
+z 0  sz   8300  num    5  bt 1300 2300 3300
+"""
+
+
+  def test_backtrace_num_field_valid(self):
+    heap = self.CreateTmpFile(self._backtrace_data)
+    backtraces, mapppings = native_heapdump_viewer.ParseNativeHeap(heap, False, True)
+    self.assertTrue(backtraces)
+    self.assertEqual(2, len(backtraces))
+
+    self.assertFalse(backtraces[0].is_zygote)
+    self.assertEqual(1000, backtraces[0].size)
+    self.assertEqual(4, backtraces[0].num_allocs)
+    self.assertEqual([0x1000, 0x2000, 0x3000], backtraces[0].frames)
+
+    self.assertTrue(backtraces[1].is_zygote)
+    self.assertEqual(8300, backtraces[1].size)
+    self.assertEqual(5, backtraces[1].num_allocs)
+    self.assertEqual([0x1300, 0x2300, 0x3300], backtraces[1].frames)
+
+  def test_backtrace_num_field_invalid(self):
+    heap = self.CreateTmpFile(self._backtrace_data)
+    backtraces, mapppings = native_heapdump_viewer.ParseNativeHeap(heap, False, False)
+    self.assertTrue(backtraces)
+    self.assertEqual(2, len(backtraces))
+
+    self.assertFalse(backtraces[0].is_zygote)
+    self.assertEqual(1000, backtraces[0].size)
+    self.assertEqual(1, backtraces[0].num_allocs)
+    self.assertEqual([0x1000, 0x2000, 0x3000], backtraces[0].frames)
+
+    self.assertTrue(backtraces[1].is_zygote)
+    self.assertEqual(8300, backtraces[1].size)
+    self.assertEqual(1, backtraces[1].num_allocs)
+    self.assertEqual([0x1300, 0x2300, 0x3300], backtraces[1].frames)
+
+  def test_backtrace_reverse_field_valid(self):
+    heap = self.CreateTmpFile(self._backtrace_data)
+    backtraces, mapppings = native_heapdump_viewer.ParseNativeHeap(heap, True, True)
+    self.assertTrue(backtraces)
+    self.assertEqual(2, len(backtraces))
+
+    self.assertFalse(backtraces[0].is_zygote)
+    self.assertEqual(1000, backtraces[0].size)
+    self.assertEqual(4, backtraces[0].num_allocs)
+    self.assertEqual([0x3000, 0x2000, 0x1000], backtraces[0].frames)
+
+    self.assertTrue(backtraces[1].is_zygote)
+    self.assertEqual(8300, backtraces[1].size)
+    self.assertEqual(5, backtraces[1].num_allocs)
+    self.assertEqual([0x3300, 0x2300, 0x1300], backtraces[1].frames)
+
+  def test_mappings(self):
+    map_data = """
+MAPS
+1000-4000 r-xp 00000000 fd:00 495    /system/lib64/libc.so
+6000-8000 r-xp 00000000 fd:00 495
+a000-f000 r-xp 0000b000 fd:00 495    /system/lib64/libutils.so
+END
+"""
+
+    heap = self.CreateTmpFile(map_data)
+    backtraces, mappings = native_heapdump_viewer.ParseNativeHeap(heap, True, True)
+
+    self.assertTrue(mappings)
+    self.assertEqual(2, len(mappings))
+
+    self.assertEqual(0x1000, mappings[0].start)
+    self.assertEqual(0x4000, mappings[0].end)
+    self.assertEqual(0, mappings[0].offset)
+    self.assertEqual("/system/lib64/libc.so", mappings[0].name)
+
+    self.assertEqual(0xa000, mappings[1].start)
+    self.assertEqual(0xf000, mappings[1].end)
+    self.assertEqual(0xb000, mappings[1].offset)
+    self.assertEqual("/system/lib64/libutils.so", mappings[1].name)
+
+if __name__ == '__main__':
+  unittest.main()