From 9a06a92a35a3bc9db17962689f9180c42fe2b785 Mon Sep 17 00:00:00 2001 From: lishutong Date: Fri, 9 Jul 2021 04:10:05 +0000 Subject: [PATCH] Add support to analyse the disk usage by file extensions. An Android AB OTA-package provide installation operations by their operation types, block adresses and payloads. One cannot know which file is being operated by an installation operation unless checking the .map file in the target build. Now, the OTA_analysis tool can analyse which file is being operated and do statistics over the the filename extensions when provided the target build. This is done by building a hashtable according to the .map file in the target build, and then query this hashtable by the operated blocks, which is defined in the OTA package. In the future, we can use segment tree instead of hashtable for better query performance. Test: Mannual tested, unit test will be added in a seperate CL. Change-Id: I150677ff81c79813ff13bf96b6401dac01e4e17a --- .../src/components/PayloadComposition.vue | 32 ++++- tools/ota_analysis/src/services/map_parser.js | 119 ++++++++++++++++++ .../src/services/payload_composition.js | 114 +++++++++++++++-- 3 files changed, 253 insertions(+), 12 deletions(-) create mode 100644 tools/ota_analysis/src/services/map_parser.js diff --git a/tools/ota_analysis/src/components/PayloadComposition.vue b/tools/ota_analysis/src/components/PayloadComposition.vue index fe8b16a31..fff3d6348 100644 --- a/tools/ota_analysis/src/components/PayloadComposition.vue +++ b/tools/ota_analysis/src/components/PayloadComposition.vue @@ -12,6 +12,16 @@ + +
@@ -20,6 +30,7 @@ diff --git a/tools/ota_analysis/src/services/map_parser.js b/tools/ota_analysis/src/services/map_parser.js new file mode 100644 index 000000000..f19e24f45 --- /dev/null +++ b/tools/ota_analysis/src/services/map_parser.js @@ -0,0 +1,119 @@ +/** + * @fileoverview Class MapParser will take in a Android build and construct + * several file name maps (physical address: file name) according to it. + * The map of each partitions is added by calling MapParser.add(partitionName). + * You can query the file name being operated by calling + * MapParser.query(address, datalength). + */ + +import * as zip from '@zip.js/zip.js/dist/zip-full.min.js' + +export class MapParser { + /** + * This class will take in a .zip Android build and construct a file type map + * @param {File} targetFile + */ + constructor(targetFile) { + this.build = new zip.ZipReader(new zip.BlobReader(targetFile)) + this.mapFiles = new Map() + this.maps = new Map() + } + + /** + * Find the .map entries in the .zip build file. Store them as a map with + * pairs of (partition name: zip.js entry). + */ + async init() { + let /** Array */ entries = await this.build.getEntries() + const /** RegExp*/ regexPath = /IMAGES\/[a-z_]*\.map/g; + const /** RegExp*/ regexName = /[\w_]+(?=\.map)/g + entries.forEach((entry) => { + if (entry.filename.match(regexPath)) { + this.mapFiles.set(entry.filename.match(regexName)[0], entry) + } + }); + } + + /** + * According to the .map in the build, build a map for later query. + * @param {String} partitionName + * @param {Number} totalLength + */ + async add(partitionName, totalLength) { + let /** Array */ map = [] + const /** RegExp */ regexNumber = /(? */fileEntries = mapText.split('\n') + // Each line of the .map file in Android build starts with the filename + // Followed by the block address, either a number or a range, for example: + // //system/apex/com.android.adbd.apex 54-66 66 66-2663 + for (let entry of fileEntries) { + let /** Array */ elements = entry.split(' ') + for (let j = 1; j < elements.length; j++) { + let /** Number */ left = 0 + let /** Number */ right = 0 + if (elements[j].match(regexRange)) { + left = parseInt(elements[j].match(/\d+/g)[0]) + right = parseInt(elements[j].match(/\d+/g)[1]) + } else { + left = parseInt(elements[j].match(regexNumber)) + right = parseInt(elements[j].match(regexNumber)) + } + InsertMap(map, elements[0], left, right) + } + } + this.maps.set(partitionName, map) + } + else { + this.maps.set(partitionName, map) + } + } + + /** + * Return the filename of given address. + * @param {String} partitionName + * @param {Array} extents + * @return {Array} + */ + query(partitionName, extents) { + let /** Array */ names = [] + let /** Array */ map = this.maps.get(partitionName) + for (let ext of extents) { + names.push(queryMap(map, + ext.startBlock, + ext.startBlock + ext.numBlocks)) + } + return names + } +} + +/** + * Fill in the hashtable from to using . + * @param {Array} map + * @param {String} name + * @param {Number} left + * @param {Number} right + */ +function InsertMap(map, name, left, right) { + for (let i = left; i <= right; i++) { + map[i] = name + } +} + +/** + * Query the hashtable using index
. + * @param {Array} map + * @param {Number} left + * @param {Number} right + */ +function queryMap(map, left, right) { + // Assuming the consecutive blocks belong to the same file + // Only the start block is queried here. + return map[left] +} \ No newline at end of file diff --git a/tools/ota_analysis/src/services/payload_composition.js b/tools/ota_analysis/src/services/payload_composition.js index 96b3c31b7..08005356d 100644 --- a/tools/ota_analysis/src/services/payload_composition.js +++ b/tools/ota_analysis/src/services/payload_composition.js @@ -5,14 +5,15 @@ */ import { OpType, MergeOpType } from '@/services/payload.js' -import { EchartsData } from '../services/echarts_data.js' +import { EchartsData } from '@/services/echarts_data.js' +import { MapParser } from '@/services/map_parser.js' /** * Add a to a element associated to . If the element dose not * exists than its value will be initialized to zero. * @param {Map} map * @param {String} key - * @param {Nynber} value + * @param {Number} value */ function addNumberToMap(map, key, value) { if (!map.get(key)) { @@ -55,12 +56,13 @@ export function mergeOperationStatistics(partitions, blockSize) { operationType, operation.dstExtent.numBlocks) } - totalBlocks += partition.newPartitionInfo.size / blockSize + // The total blocks number should be rounded up + totalBlocks += Math.ceil(partition.newPartitionInfo.size / blockSize) } // The COW merge operation is default to be COW_replace and not shown in // the manifest info. We have to mannually add that part of operations, // by subtracting the total blocks with other blocks. - mergeOperations.forEach((value, key)=> totalBlocks -= value ) + mergeOperations.forEach((value, key) => totalBlocks -= value) mergeOperations.set('COW_REPLACE', totalBlocks) return mergeOperations } @@ -87,13 +89,55 @@ export function operatedPayloadStatistics(partitions) { return operatedBlocks } +/** + * Return a statistics over the disk usage of each file types in a OTA package. + * A target file has to be provided and address-filename maps will be built. + * Only partitions that are being passed in will be included. + * @param {Array} partitions + * @param {Number} blockSize + * @param {File} targetFile + * @return {Map} + */ +export async function operatedExtensionStatistics(partitions, blockSize, targetFile) { + let /** Map */ operatedExtensions = new Map() + if (!targetFile) { + return operatedExtensions + } + let buildMap = new MapParser(targetFile) + await buildMap.init() + for (let partition of partitions) { + await buildMap.add( + partition.partitionName, + Math.ceil(partition.newPartitionInfo.size / blockSize)) + for (let operation of partition.operations) { + if (!operation.hasOwnProperty('dataLength')) continue + let operatedFileNames = buildMap.query( + partition.partitionName, + operation.dstExtents) + let extentDataLength = distributeExtensions( + operatedFileNames, + operation.dstExtents, + operation.dataLength + ) + extentDataLength.forEach((value, key) => { + addNumberToMap( + operatedExtensions, + key, + value + ) + }) + } + } + return operatedExtensions +} + /** * Analyse the given partitions using the given metrics. * @param {String} metrics * @param {Array} partitions * @return {EchartsData} */ -export function analysePartitions(metrics, partitions, blockSize=4096) { +export async function analysePartitions(metrics, partitions, blockSize = 4096, targetFile = null) { let /** Map */statisticsData let /** Echartsdata */ echartsData switch (metrics) { @@ -120,8 +164,25 @@ export function analysePartitions(metrics, partitions, blockSize=4096) { 'COW merge operations', 'blocks' ) + break + case 'extensions': + try { + statisticsData = await operatedExtensionStatistics(partitions, blockSize, targetFile) + } + catch (err) { + throw err + } + echartsData = new EchartsData( + statisticsData, + 'Size of operated filename extensions', + 'bytes' + ) + } + if (echartsData) { + return echartsData + } else { + throw 'Please double check if this is a proper AB OTA package.' } - return echartsData } /** @@ -137,11 +198,50 @@ export function numBlocks(exts) { /** * Return a string that indicates the blocks being operated * in the manner of (start_block, block_length) - * @param {Array} exts * @return {string} */ export function displayBlocks(exts) { const accumulator = (total, ext) => total + '(' + ext.startBlock + ',' + ext.numBlocks + ')' return exts.reduce(accumulator, '') +} + +/** + * Return a map with pairs of (file extension, data length used by this + * extension). The total data length will be distributed by the blocks ratio + * of each extent. + * @param {Array} filenames + * @param {Array} exts + * @param {Number} length + * @return {Map} + */ +export function distributeExtensions(filenames, exts, length) { + let totalBlocks = numBlocks(exts) + let distributedLengths = new Map() + for (let i = 0; i < filenames.length; i++) { + addNumberToMap( + distributedLengths, + name2Extension(filenames[i]), + Math.round(length * exts[i].numBlocks / totalBlocks) + ) + } + return distributedLengths +} + +/** + * convert a filename into extension, for example: + * '//system/apex/com.android.adbd.apex' => 'apex' + * @param {String} filename + * @return {String} + */ +export function name2Extension(filename) { + let elements = filename.split('.') + if (elements.length>1) { + return elements[elements.length - 1] + } else if (elements[0]==='unknown') { + return 'unknown' + } else { + return 'no-extension' + } } \ No newline at end of file