Add support to analyse the disk usage by file extensions.

An Android AB OTA-package provide installation operations by their
operation types, block adresses and payloads. One cannot know which file
is being operated by an installation operation unless checking the .map
file in the target build.

Now, the OTA_analysis tool can analyse which file is being operated and
do statistics over the the filename extensions when provided the target
build. This is done by building a hashtable according to the .map file
in the target build, and then query this hashtable by the operated
blocks, which is defined in the OTA package.

In the future, we can use segment tree instead of hashtable for better
query performance.

Test: Mannual tested, unit test will be added in a seperate CL.
Change-Id: I150677ff81c79813ff13bf96b6401dac01e4e17a

tools/ota_analysis/src/components/PayloadComposition.vue

@@ -12,6 +12,16 @@
   <button @click="updateChart('COWmerge')">
     Analyse COW Merge Operations
   </button>
+  <BaseFile
+    label="Select The Target Android Build"
+    @file-select="selectBuild"
+  />
+  <button
+    :disabled="!targetFile"
+    @click="updateChart('extensions')"
+  >
+    Analyse File Extensions
+  </button>
   <div v-if="echartsData">
     <PieChart :echartsData="echartsData" />
   </div>
@@ -20,6 +30,7 @@
 <script>
 import PartialCheckbox from '@/components/PartialCheckbox.vue'
 import PieChart from '@/components/PieChart.vue'
+import BaseFile from '@/components/BaseFile.vue'
 import { analysePartitions } from '../services/payload_composition.js'
 import { chromeos_update_engine as update_metadata_pb } from '../services/update_metadata_pb.js'
 
@@ -27,6 +38,7 @@ export default {
   components: {
     PartialCheckbox,
     PieChart,
+    BaseFile
   },
   props: {
     manifest: {
@@ -39,6 +51,7 @@ export default {
       partitionInclude: new Map(),
       echartsData: null,
       listData: '',
+      targetFile: null
     }
   },
   computed: {
@@ -49,15 +62,24 @@ export default {
     },
   },
   methods: {
-    updateChart(metrics) {
+    async updateChart(metrics) {
       let partitionSelected = this.manifest.partitions.filter((partition) =>
         this.partitionInclude.get(partition.partitionName)
       )
-      this.echartsData = analysePartitions(
+      try {
+        this.echartsData = await analysePartitions(
           metrics,
           partitionSelected,
-        this.manifest.blockSize)
+          this.manifest.blockSize,
+          this.targetFile) }
+      catch (err) {
+        alert('Cannot be processed for the following issue: ', err)
+      }
     },
+    selectBuild(files) {
+      //TODO(lishutong) check the version of target file is same to the OTA target
+      this.targetFile = files[0]
+    }
   },
 }
 </script>

tools/ota_analysis/src/services/map_parser.js

@@ -0,0 +1,119 @@
+/**
+ * @fileoverview Class MapParser will take in a Android build and construct
+ * several file name maps (physical address: file name) according to it.
+ * The map of each partitions is added by calling MapParser.add(partitionName).
+ * You can query the file name being operated by calling
+ * MapParser.query(address, datalength).
+ */
+
+import * as zip from '@zip.js/zip.js/dist/zip-full.min.js'
+
+export class MapParser {
+  /**
+   * This class will take in a .zip Android build and construct a file type map
+   * @param {File} targetFile
+   */
+  constructor(targetFile) {
+    this.build = new zip.ZipReader(new zip.BlobReader(targetFile))
+    this.mapFiles = new Map()
+    this.maps = new Map()
+  }
+
+  /**
+   * Find the .map entries in the .zip build file. Store them as a map with
+   * pairs of (partition name: zip.js entry).
+   */
+  async init() {
+    let /** Array<Entry> */ entries = await this.build.getEntries()
+    const /** RegExp*/ regexPath = /IMAGES\/[a-z_]*\.map/g;
+    const /** RegExp*/ regexName = /[\w_]+(?=\.map)/g
+    entries.forEach((entry) => {
+      if (entry.filename.match(regexPath)) {
+        this.mapFiles.set(entry.filename.match(regexName)[0], entry)
+      }
+    });
+  }
+
+  /**
+   * According to the .map in the build, build a map for later query.
+   * @param {String} partitionName
+   * @param {Number} totalLength
+   */
+  async add(partitionName, totalLength) {
+    let /** Array<String> */ map = []
+    const /** RegExp */ regexNumber = /(?<![0-9\-])\d+(?![0-9\-])/g
+    const /** Reg */ regexRange = /\d+\-\d+/g
+    for (let i = 0; i < totalLength; i++) map[i] = 'unknown'
+    if (this.mapFiles.get(partitionName)) {
+      let /** String */mapText =
+        await this.mapFiles.get(partitionName).getData(
+          new zip.TextWriter()
+        )
+      let /** Array<String> */fileEntries = mapText.split('\n')
+      // Each line of the .map file in Android build starts with the filename
+      // Followed by the block address, either a number or a range, for example:
+      // //system/apex/com.android.adbd.apex 54-66 66 66-2663
+      for (let entry of fileEntries) {
+        let /** Array<String> */ elements = entry.split(' ')
+        for (let j = 1; j < elements.length; j++) {
+          let /** Number */ left = 0
+          let /** Number */ right = 0
+          if (elements[j].match(regexRange)) {
+            left = parseInt(elements[j].match(/\d+/g)[0])
+            right = parseInt(elements[j].match(/\d+/g)[1])
+          } else {
+            left = parseInt(elements[j].match(regexNumber))
+            right = parseInt(elements[j].match(regexNumber))
+          }
+          InsertMap(map, elements[0], left, right)
+        }
+      }
+      this.maps.set(partitionName, map)
+    }
+    else {
+      this.maps.set(partitionName, map)
+    }
+  }
+
+  /**
+   * Return the filename of given address.
+   * @param {String} partitionName
+   * @param {Array<PartitionUpdate>} extents
+   * @return {Array<String>}
+   */
+  query(partitionName, extents) {
+    let /** Array<String> */ names = []
+    let /** Array<String> */ map = this.maps.get(partitionName)
+    for (let ext of extents) {
+      names.push(queryMap(map,
+        ext.startBlock,
+        ext.startBlock + ext.numBlocks))
+    }
+    return names
+  }
+}
+
+/**
+ * Fill in the hashtable from <left> to <right> using <name>.
+ * @param {Array<String>} map
+ * @param {String} name
+ * @param {Number} left
+ * @param {Number} right
+ */
+function InsertMap(map, name, left, right) {
+  for (let i = left; i <= right; i++) {
+    map[i] = name
+  }
+}
+
+/**
+ * Query the hashtable <map> using index <address>.
+ * @param {Array<String>} map
+ * @param {Number} left
+ * @param {Number} right
+ */
+function queryMap(map, left, right) {
+  // Assuming the consecutive blocks belong to the same file
+  // Only the start block is queried here.
+  return map[left]
+}

tools/ota_analysis/src/services/payload_composition.js

@@ -5,14 +5,15 @@
  */
 
 import { OpType, MergeOpType } from '@/services/payload.js'
-import { EchartsData } from '../services/echarts_data.js'
+import { EchartsData } from '@/services/echarts_data.js'
+import { MapParser } from '@/services/map_parser.js'
 
 /**
  * Add a <value> to a element associated to <key>. If the element dose not
  * exists than its value will be initialized to zero.
  * @param {Map} map
  * @param {String} key
- * @param {Nynber} value
+ * @param {Number} value
  */
 function addNumberToMap(map, key, value) {
   if (!map.get(key)) {
@@ -55,12 +56,13 @@ export function mergeOperationStatistics(partitions, blockSize) {
         operationType,
         operation.dstExtent.numBlocks)
     }
-    totalBlocks += partition.newPartitionInfo.size / blockSize
+    // The total blocks number should be rounded up
+    totalBlocks += Math.ceil(partition.newPartitionInfo.size / blockSize)
   }
   // The COW merge operation is default to be COW_replace and not shown in
   // the manifest info. We have to mannually add that part of operations,
   // by subtracting the total blocks with other blocks.
-  mergeOperations.forEach((value, key)=> totalBlocks -= value )
+  mergeOperations.forEach((value, key) => totalBlocks -= value)
   mergeOperations.set('COW_REPLACE', totalBlocks)
   return mergeOperations
 }
@@ -87,13 +89,55 @@ export function operatedPayloadStatistics(partitions) {
   return operatedBlocks
 }
 
+/**
+ * Return a statistics over the disk usage of each file types in a OTA package.
+ * A target file has to be provided and address-filename maps will be built.
+ * Only partitions that are being passed in will be included.
+ * @param {Array<PartitionUpdate>} partitions
+ * @param {Number} blockSize
+ * @param {File} targetFile
+ * @return {Map}
+ */
+export async function operatedExtensionStatistics(partitions, blockSize, targetFile) {
+  let /** Map */ operatedExtensions = new Map()
+  if (!targetFile) {
+    return operatedExtensions
+  }
+  let buildMap = new MapParser(targetFile)
+  await buildMap.init()
+  for (let partition of partitions) {
+    await buildMap.add(
+      partition.partitionName,
+      Math.ceil(partition.newPartitionInfo.size / blockSize))
+    for (let operation of partition.operations) {
+      if (!operation.hasOwnProperty('dataLength')) continue
+      let operatedFileNames = buildMap.query(
+        partition.partitionName,
+        operation.dstExtents)
+      let extentDataLength = distributeExtensions(
+        operatedFileNames,
+        operation.dstExtents,
+        operation.dataLength
+      )
+      extentDataLength.forEach((value, key) => {
+        addNumberToMap(
+          operatedExtensions,
+          key,
+          value
+        )
+      })
+    }
+  }
+  return operatedExtensions
+}
+
 /**
  * Analyse the given partitions using the given metrics.
  * @param {String} metrics
  * @param {Array<PartitionUpdate>} partitions
  * @return {EchartsData}
  */
-export function analysePartitions(metrics, partitions, blockSize=4096) {
+export async function analysePartitions(metrics, partitions, blockSize = 4096, targetFile = null) {
   let /** Map */statisticsData
   let /** Echartsdata */ echartsData
   switch (metrics) {
@@ -120,8 +164,25 @@ export function analysePartitions(metrics, partitions, blockSize=4096) {
       'COW merge operations',
       'blocks'
     )
+    break
+  case 'extensions':
+    try {
+      statisticsData = await operatedExtensionStatistics(partitions, blockSize, targetFile)
     }
+    catch (err) {
+      throw err
+    }
+    echartsData = new EchartsData(
+      statisticsData,
+      'Size of operated filename extensions',
+      'bytes'
+    )
+  }
+  if (echartsData) {
     return echartsData
+  } else {
+    throw 'Please double check if this is a proper AB OTA package.'
+  }
 }
 
 /**
@@ -137,7 +198,7 @@ export function numBlocks(exts) {
 /**
  * Return a string that indicates the blocks being operated
  * in the manner of (start_block, block_length)
- * @param {Array<InstallOperations} exts
+ * @param {Array<InstallOperations>} exts
  * @return {string}
  */
 export function displayBlocks(exts) {
@@ -145,3 +206,42 @@ export function displayBlocks(exts) {
     total + '(' + ext.startBlock + ',' + ext.numBlocks + ')'
   return exts.reduce(accumulator, '')
 }
+
+/**
+ * Return a map with pairs of (file extension, data length used by this
+ * extension). The total data length will be distributed by the blocks ratio
+ * of each extent.
+ * @param {Array<String>} filenames
+ * @param {Array<InstallOperations>} exts
+ * @param {Number} length
+ * @return {Map}
+ */
+export function distributeExtensions(filenames, exts, length) {
+  let totalBlocks = numBlocks(exts)
+  let distributedLengths = new Map()
+  for (let i = 0; i < filenames.length; i++) {
+    addNumberToMap(
+      distributedLengths,
+      name2Extension(filenames[i]),
+      Math.round(length * exts[i].numBlocks / totalBlocks)
+    )
+  }
+  return distributedLengths
+}
+
+/**
+ * convert a filename into extension, for example:
+ * '//system/apex/com.android.adbd.apex' => 'apex'
+ * @param {String} filename
+ * @return {String}
+ */
+export function name2Extension(filename) {
+  let elements = filename.split('.')
+  if (elements.length>1) {
+    return elements[elements.length - 1]
+  } else if (elements[0]==='unknown') {
+    return 'unknown'
+  } else {
+    return 'no-extension'
+  }
+}