Merge changes I84bfadd3,I150677ff

* changes:
  Avoid showing too many entries in the pie chart once.
  Add support to analyse the disk usage by file extensions.

tools/ota_analysis/src/components/PayloadComposition.vue

@@ -12,6 +12,16 @@
   <button @click="updateChart('COWmerge')">
     Analyse COW Merge Operations
   </button>
+  <BaseFile
+    label="Select The Target Android Build"
+    @file-select="selectBuild"
+  />
+  <button
+    :disabled="!targetFile"
+    @click="updateChart('extensions')"
+  >
+    Analyse File Extensions
+  </button>
   <div v-if="echartsData">
     <PieChart :echartsData="echartsData" />
   </div>
@@ -20,6 +30,7 @@
 <script>
 import PartialCheckbox from '@/components/PartialCheckbox.vue'
 import PieChart from '@/components/PieChart.vue'
+import BaseFile from '@/components/BaseFile.vue'
 import { analysePartitions } from '../services/payload_composition.js'
 import { chromeos_update_engine as update_metadata_pb } from '../services/update_metadata_pb.js'
 
@@ -27,6 +38,7 @@ export default {
   components: {
     PartialCheckbox,
     PieChart,
+    BaseFile
   },
   props: {
     manifest: {
@@ -39,6 +51,7 @@ export default {
       partitionInclude: new Map(),
       echartsData: null,
       listData: '',
+      targetFile: null
     }
   },
   computed: {
@@ -49,15 +62,24 @@ export default {
     },
   },
   methods: {
-    updateChart(metrics) {
+    async updateChart(metrics) {
       let partitionSelected = this.manifest.partitions.filter((partition) =>
         this.partitionInclude.get(partition.partitionName)
       )
-      this.echartsData = analysePartitions(
-        metrics,
-        partitionSelected,
-        this.manifest.blockSize)
+      try {
+        this.echartsData = await analysePartitions(
+          metrics,
+          partitionSelected,
+          this.manifest.blockSize,
+          this.targetFile) }
+      catch (err) {
+        alert('Cannot be processed for the following issue: ', err)
+      }
     },
+    selectBuild(files) {
+      //TODO(lishutong) check the version of target file is same to the OTA target
+      this.targetFile = files[0]
+    }
   },
 }
 </script>

tools/ota_analysis/src/services/echarts_data.js

@@ -5,11 +5,13 @@ export class EchartsData {
    * @param {Map} statisticData
    * @param {String} title
    * @param {String} unit
+   * @param {Number} maximumEntries
    */
-  constructor(statisticData, title, unit) {
+  constructor(statisticData, title, unit, maximumEntries = 15) {
     this.statisticData = statisticData
     this.title = title
     this.unit = unit
+    this.maximumEntries = maximumEntries
   }
 
   /**
@@ -32,6 +34,9 @@ export class EchartsData {
    * @return {Object} an ECharts option object.
    */
   getEchartsOption() {
+    if (this.statisticData.size > this.maximumEntries) {
+      this.statisticData = trimMap(this.statisticData, this.maximumEntries)
+    }
     let /** Object */ option = new Object()
     option.title = {
       text: this.title,
@@ -67,4 +72,39 @@ export class EchartsData {
     ]
     return option
   }
+}
+
+/**
+ * When there are too many entries in the map, the pie chart can be very
+ * crowded. This function will return the entries that have high values.
+ * Specifically, the top <maximumEntries> will be stored and the others
+ * will be added into an entry called 'other'.
+ * @param {Map} map
+ * @param {Number} maximumEntries
+ * @return {Map}
+ */
+function trimMap(map, maximumEntries) {
+  if (map.size <= maximumEntries) return map
+  let /** Map */ new_map = new Map()
+  for (let i=0; i<maximumEntries; i++) {
+    let /** Number */ curr = 0
+    let /** String */ currKey = ''
+    for (let [key, value] of map) {
+      if (!new_map.get(key)) {
+        if (value > curr) {
+          curr = value
+          currKey = key
+        }
+      }
+    }
+    new_map.set(currKey, curr)
+  }
+  let /** Number */ restTotal = 0
+  for (let [key, value] of map) {
+    if (!new_map.get(key)) {
+      restTotal += value
+    }
+  }
+  new_map.set('other', restTotal)
+  return new_map
 }

tools/ota_analysis/src/services/map_parser.js

@@ -0,0 +1,119 @@
+/**
+ * @fileoverview Class MapParser will take in a Android build and construct
+ * several file name maps (physical address: file name) according to it.
+ * The map of each partitions is added by calling MapParser.add(partitionName).
+ * You can query the file name being operated by calling
+ * MapParser.query(address, datalength).
+ */
+
+import * as zip from '@zip.js/zip.js/dist/zip-full.min.js'
+
+export class MapParser {
+  /**
+   * This class will take in a .zip Android build and construct a file type map
+   * @param {File} targetFile
+   */
+  constructor(targetFile) {
+    this.build = new zip.ZipReader(new zip.BlobReader(targetFile))
+    this.mapFiles = new Map()
+    this.maps = new Map()
+  }
+
+  /**
+   * Find the .map entries in the .zip build file. Store them as a map with
+   * pairs of (partition name: zip.js entry).
+   */
+  async init() {
+    let /** Array<Entry> */ entries = await this.build.getEntries()
+    const /** RegExp*/ regexPath = /IMAGES\/[a-z_]*\.map/g;
+    const /** RegExp*/ regexName = /[\w_]+(?=\.map)/g
+    entries.forEach((entry) => {
+      if (entry.filename.match(regexPath)) {
+        this.mapFiles.set(entry.filename.match(regexName)[0], entry)
+      }
+    });
+  }
+
+  /**
+   * According to the .map in the build, build a map for later query.
+   * @param {String} partitionName
+   * @param {Number} totalLength
+   */
+  async add(partitionName, totalLength) {
+    let /** Array<String> */ map = []
+    const /** RegExp */ regexNumber = /(?<![0-9\-])\d+(?![0-9\-])/g
+    const /** Reg */ regexRange = /\d+\-\d+/g
+    for (let i = 0; i < totalLength; i++) map[i] = 'unknown'
+    if (this.mapFiles.get(partitionName)) {
+      let /** String */mapText =
+        await this.mapFiles.get(partitionName).getData(
+          new zip.TextWriter()
+        )
+      let /** Array<String> */fileEntries = mapText.split('\n')
+      // Each line of the .map file in Android build starts with the filename
+      // Followed by the block address, either a number or a range, for example:
+      // //system/apex/com.android.adbd.apex 54-66 66 66-2663
+      for (let entry of fileEntries) {
+        let /** Array<String> */ elements = entry.split(' ')
+        for (let j = 1; j < elements.length; j++) {
+          let /** Number */ left = 0
+          let /** Number */ right = 0
+          if (elements[j].match(regexRange)) {
+            left = parseInt(elements[j].match(/\d+/g)[0])
+            right = parseInt(elements[j].match(/\d+/g)[1])
+          } else {
+            left = parseInt(elements[j].match(regexNumber))
+            right = parseInt(elements[j].match(regexNumber))
+          }
+          InsertMap(map, elements[0], left, right)
+        }
+      }
+      this.maps.set(partitionName, map)
+    }
+    else {
+      this.maps.set(partitionName, map)
+    }
+  }
+
+  /**
+   * Return the filename of given address.
+   * @param {String} partitionName
+   * @param {Array<PartitionUpdate>} extents
+   * @return {Array<String>}
+   */
+  query(partitionName, extents) {
+    let /** Array<String> */ names = []
+    let /** Array<String> */ map = this.maps.get(partitionName)
+    for (let ext of extents) {
+      names.push(queryMap(map,
+        ext.startBlock,
+        ext.startBlock + ext.numBlocks))
+    }
+    return names
+  }
+}
+
+/**
+ * Fill in the hashtable from <left> to <right> using <name>.
+ * @param {Array<String>} map
+ * @param {String} name
+ * @param {Number} left
+ * @param {Number} right
+ */
+function InsertMap(map, name, left, right) {
+  for (let i = left; i <= right; i++) {
+    map[i] = name
+  }
+}
+
+/**
+ * Query the hashtable <map> using index <address>.
+ * @param {Array<String>} map
+ * @param {Number} left
+ * @param {Number} right
+ */
+function queryMap(map, left, right) {
+  // Assuming the consecutive blocks belong to the same file
+  // Only the start block is queried here.
+  return map[left]
+}

tools/ota_analysis/src/services/payload_composition.js

@@ -5,14 +5,15 @@
  */
 
 import { OpType, MergeOpType } from '@/services/payload.js'
-import { EchartsData } from '../services/echarts_data.js'
+import { EchartsData } from '@/services/echarts_data.js'
+import { MapParser } from '@/services/map_parser.js'
 
 /**
  * Add a <value> to a element associated to <key>. If the element dose not
  * exists than its value will be initialized to zero.
  * @param {Map} map
  * @param {String} key
- * @param {Nynber} value
+ * @param {Number} value
  */
 function addNumberToMap(map, key, value) {
   if (!map.get(key)) {
@@ -55,12 +56,13 @@ export function mergeOperationStatistics(partitions, blockSize) {
         operationType,
         operation.dstExtent.numBlocks)
     }
-    totalBlocks += partition.newPartitionInfo.size / blockSize
+    // The total blocks number should be rounded up
+    totalBlocks += Math.ceil(partition.newPartitionInfo.size / blockSize)
   }
   // The COW merge operation is default to be COW_replace and not shown in
   // the manifest info. We have to mannually add that part of operations,
   // by subtracting the total blocks with other blocks.
-  mergeOperations.forEach((value, key)=> totalBlocks -= value )
+  mergeOperations.forEach((value, key) => totalBlocks -= value)
   mergeOperations.set('COW_REPLACE', totalBlocks)
   return mergeOperations
 }
@@ -87,13 +89,55 @@ export function operatedPayloadStatistics(partitions) {
   return operatedBlocks
 }
 
+/**
+ * Return a statistics over the disk usage of each file types in a OTA package.
+ * A target file has to be provided and address-filename maps will be built.
+ * Only partitions that are being passed in will be included.
+ * @param {Array<PartitionUpdate>} partitions
+ * @param {Number} blockSize
+ * @param {File} targetFile
+ * @return {Map}
+ */
+export async function operatedExtensionStatistics(partitions, blockSize, targetFile) {
+  let /** Map */ operatedExtensions = new Map()
+  if (!targetFile) {
+    return operatedExtensions
+  }
+  let buildMap = new MapParser(targetFile)
+  await buildMap.init()
+  for (let partition of partitions) {
+    await buildMap.add(
+      partition.partitionName,
+      Math.ceil(partition.newPartitionInfo.size / blockSize))
+    for (let operation of partition.operations) {
+      if (!operation.hasOwnProperty('dataLength')) continue
+      let operatedFileNames = buildMap.query(
+        partition.partitionName,
+        operation.dstExtents)
+      let extentDataLength = distributeExtensions(
+        operatedFileNames,
+        operation.dstExtents,
+        operation.dataLength
+      )
+      extentDataLength.forEach((value, key) => {
+        addNumberToMap(
+          operatedExtensions,
+          key,
+          value
+        )
+      })
+    }
+  }
+  return operatedExtensions
+}
+
 /**
  * Analyse the given partitions using the given metrics.
  * @param {String} metrics
  * @param {Array<PartitionUpdate>} partitions
  * @return {EchartsData}
  */
-export function analysePartitions(metrics, partitions, blockSize=4096) {
+export async function analysePartitions(metrics, partitions, blockSize = 4096, targetFile = null) {
   let /** Map */statisticsData
   let /** Echartsdata */ echartsData
   switch (metrics) {
@@ -120,8 +164,25 @@ export function analysePartitions(metrics, partitions, blockSize=4096) {
       'COW merge operations',
       'blocks'
     )
+    break
+  case 'extensions':
+    try {
+      statisticsData = await operatedExtensionStatistics(partitions, blockSize, targetFile)
+    }
+    catch (err) {
+      throw err
+    }
+    echartsData = new EchartsData(
+      statisticsData,
+      'Size of operated filename extensions',
+      'bytes'
+    )
+  }
+  if (echartsData) {
+    return echartsData
+  } else {
+    throw 'Please double check if this is a proper AB OTA package.'
   }
-  return echartsData
 }
 
 /**
@@ -137,11 +198,50 @@ export function numBlocks(exts) {
 /**
  * Return a string that indicates the blocks being operated
  * in the manner of (start_block, block_length)
- * @param {Array<InstallOperations} exts
+ * @param {Array<InstallOperations>} exts
  * @return {string}
  */
 export function displayBlocks(exts) {
   const accumulator = (total, ext) =>
     total + '(' + ext.startBlock + ',' + ext.numBlocks + ')'
   return exts.reduce(accumulator, '')
+}
+
+/**
+ * Return a map with pairs of (file extension, data length used by this
+ * extension). The total data length will be distributed by the blocks ratio
+ * of each extent.
+ * @param {Array<String>} filenames
+ * @param {Array<InstallOperations>} exts
+ * @param {Number} length
+ * @return {Map}
+ */
+export function distributeExtensions(filenames, exts, length) {
+  let totalBlocks = numBlocks(exts)
+  let distributedLengths = new Map()
+  for (let i = 0; i < filenames.length; i++) {
+    addNumberToMap(
+      distributedLengths,
+      name2Extension(filenames[i]),
+      Math.round(length * exts[i].numBlocks / totalBlocks)
+    )
+  }
+  return distributedLengths
+}
+
+/**
+ * convert a filename into extension, for example:
+ * '//system/apex/com.android.adbd.apex' => 'apex'
+ * @param {String} filename
+ * @return {String}
+ */
+export function name2Extension(filename) {
+  let elements = filename.split('.')
+  if (elements.length>1) {
+    return elements[elements.length - 1]
+  } else if (elements[0]==='unknown') {
+    return 'unknown'
+  } else {
+    return 'no-extension'
+  }
 }