Merge changes Icd24f2c0,Ie1812552

* changes:
  Add support for analysing the COW merge operations in OTA packages.
  Update to new update_metadata.proto which support COW merge operation.

tools/otagui/.eslintignore

@@ -0,0 +1 @@
+src/services/update_metadata_pb.js

tools/otagui/src/components/PayloadComposition.vue

@@ -9,6 +9,9 @@
   <button @click="updateChart('payload')">
     Analyse Payload Composition
   </button>
+  <button @click="updateChart('COWmerge')">
+    Analyse COW Merge Operations
+  </button>
   <div v-if="echartsData">
     <PieChart :echartsData="echartsData" />
   </div>
@@ -17,11 +20,7 @@
 <script>
 import PartialCheckbox from '@/components/PartialCheckbox.vue'
 import PieChart from '@/components/PieChart.vue'
-import {
+import { analysePartitions } from '../services/payload_composition.js'
-  operatedBlockStatistics,
-  operatedPayloadStatistics,
-} from '../services/payload_composition.js'
-import { EchartsData } from '../services/echarts_data.js'
 import { chromeos_update_engine as update_metadata_pb } from '../services/update_metadata_pb.js'
 
 export default {
@@ -54,24 +53,10 @@ export default {
       let partitionSelected = this.manifest.partitions.filter((partition) =>
         this.partitionInclude.get(partition.partitionName)
       )
-      let statisticsData
+      this.echartsData = analysePartitions(
-      switch (metrics) {
+        metrics,
-      case 'blocks':
+        partitionSelected,
-        statisticsData = operatedBlockStatistics(partitionSelected)
+        this.manifest.blockSize)
-        this.echartsData = new EchartsData(
-          statisticsData,
-          'Operated blocks in target build'
-        )
-        break
-      case 'payload':
-        statisticsData = operatedPayloadStatistics(partitionSelected)
-        this.echartsData = new EchartsData(
-          statisticsData,
-          'Payload disk usage'
-        )
-        break
-      }
-      this.listData = this.echartsData.listData()
     },
   },
 }

tools/otagui/src/services/echarts_data.js

@@ -4,10 +4,12 @@ export class EchartsData {
    * usage in Vue-Echarts.
    * @param {Map} statisticData
    * @param {String} title
+   * @param {String} unit
    */
-  constructor(statisticData, title) {
+  constructor(statisticData, title, unit) {
     this.statisticData = statisticData
     this.title = title
+    this.unit = unit
   }
 
   /**
@@ -26,6 +28,7 @@ export class EchartsData {
    * Generate necessary parameters (option) for vue-echarts.
    * Format of the parameters can be found here:
    * https://echarts.apache.org/en/option.html
+   * @param {String} unit
    * @return {Object} an ECharts option object.
    */
   getEchartsOption() {
@@ -36,7 +39,7 @@ export class EchartsData {
     }
     option.tooltip = {
       trigger: "item",
-      formatter: "{a} <br/>{b} : {c} ({d}%)"
+      formatter: "{a} <br/>{b} : {c} " + this.unit + " ({d}%)"
     }
     option.legend = {
       orient: "horizontal",

tools/otagui/src/services/payload.js

@@ -1,5 +1,5 @@
 /**
- * @fileoverview Clss paypload is used to read in and
+ * @fileoverview Class paypload is used to read in and
  * parse the payload.bin file from a OTA.zip file.
  * Class OpType creates a Map that can resolve the
  * operation type.
@@ -128,4 +128,20 @@ export class OpType {
       this.mapType.set(types[key], key)
     }
   }
+}
+
+export class MergeOpType {
+  /**
+   * MergeOpType create a map that could resolve the COW merge operation
+   * types. This is very similar to OpType class except that one is for
+   * installation operations.
+   */
+  constructor() {
+    let /** Array<{String: Number}>*/ types =
+      update_metadata_pb.CowMergeOperation.Type
+    this.mapType = new Map()
+    for (let key in types) {
+      this.mapType.set(types[key], key)
+    }
+  }
 }

tools/otagui/src/services/payload_composition.js

@@ -1,4 +1,25 @@
-import { OpType } from '@/services/payload.js'
+/**
+ * @fileoverview Offer functions that can be used to parse the partitionUpdate
+ * and then do statistics over it. One can use analysePartitions to specify the
+ * partitions been analysed and metrics.
+ */
+
+import { OpType, MergeOpType } from '@/services/payload.js'
+import { EchartsData } from '../services/echarts_data.js'
+
+/**
+ * Add a <value> to a element associated to <key>. If the element dose not
+ * exists than its value will be initialized to zero.
+ * @param {Map} map
+ * @param {String} key
+ * @param {Nynber} value
+ */
+function addNumberToMap(map, key, value) {
+  if (!map.get(key)) {
+    map.set(key, 0)
+  }
+  map.set(key, map.get(key) + value)
+}
 
 /**
  * Return a statistics over the numbers of blocks (in destination) that are
@@ -8,23 +29,42 @@ import { OpType } from '@/services/payload.js'
  * @return {Map}
  */
 export function operatedBlockStatistics(partitions) {
-  let operatedBlocks = new Map()
+  let /** Map */ operatedBlocks = new Map()
-  let opType = new OpType()
+  let /** OpType */ opType = new OpType()
   for (let partition of partitions) {
     for (let operation of partition.operations) {
       let operationType = opType.mapType.get(operation.type)
-      if (!operatedBlocks.get(operationType)) {
+      addNumberToMap(
-        operatedBlocks.set(operationType, 0)
+        operatedBlocks,
-      }
-      operatedBlocks.set(
         operationType,
-        operatedBlocks.get(operationType) + numBlocks(operation.dstExtents)
+        numBlocks(operation.dstExtents))
-      )
     }
   }
   return operatedBlocks
 }
 
+export function mergeOperationStatistics(partitions, blockSize) {
+  let /** Map */ mergeOperations = new Map()
+  let /** MergeOpType */ opType = new MergeOpType()
+  let /** Number */ totalBlocks = 0
+  for (let partition of partitions) {
+    for (let operation of partition.mergeOperations) {
+      let operationType = opType.mapType.get(operation.type)
+      addNumberToMap(
+        mergeOperations,
+        operationType,
+        operation.dstExtent.numBlocks)
+    }
+    totalBlocks += partition.newPartitionInfo.size / blockSize
+  }
+  // The COW merge operation is default to be COW_replace and not shown in
+  // the manifest info. We have to mannually add that part of operations,
+  // by subtracting the total blocks with other blocks.
+  mergeOperations.forEach((value, key)=> totalBlocks -= value )
+  mergeOperations.set('COW_REPLACE', totalBlocks)
+  return mergeOperations
+}
+
 /**
  * Return a statistics over the disk usage of payload.bin, based on the type of
  * installation operations. Only partitions that are being passed in will be
@@ -33,23 +73,57 @@ export function operatedBlockStatistics(partitions) {
  * @return {Map}
  */
 export function operatedPayloadStatistics(partitions) {
-  let operatedBlocks = new Map()
+  let /** Map */ operatedBlocks = new Map()
-  let opType = new OpType()
+  let /** OpType */ opType = new OpType()
   for (let partition of partitions) {
     for (let operation of partition.operations) {
       let operationType = opType.mapType.get(operation.type)
-      if (!operatedBlocks.get(operationType)) {
+      addNumberToMap(
-        operatedBlocks.set(operationType, 0)
+        operatedBlocks,
-      }
-      operatedBlocks.set(
         operationType,
-        operatedBlocks.get(operationType) + operation.dataLength
+        operation.dataLength)
-      )
     }
   }
   return operatedBlocks
 }
 
+/**
+ * Analyse the given partitions using the given metrics.
+ * @param {String} metrics
+ * @param {Array<PartitionUpdate>} partitions
+ * @return {EchartsData}
+ */
+export function analysePartitions(metrics, partitions, blockSize=4096) {
+  let /** Map */statisticsData
+  let /** Echartsdata */ echartsData
+  switch (metrics) {
+  case 'blocks':
+    statisticsData = operatedBlockStatistics(partitions)
+    echartsData = new EchartsData(
+      statisticsData,
+      'Operated blocks in target build',
+      'blocks'
+    )
+    break
+  case 'payload':
+    statisticsData = operatedPayloadStatistics(partitions)
+    echartsData = new EchartsData(
+      statisticsData,
+      'Payload disk usage',
+      'bytes'
+    )
+    break
+  case 'COWmerge':
+    statisticsData = mergeOperationStatistics(partitions, blockSize)
+    echartsData = new EchartsData(
+      statisticsData,
+      'COW merge operations',
+      'blocks'
+    )
+  }
+  return echartsData
+}
+
 /**
  * Calculate the number of blocks being operated
  * @param {Array<InstallOperations>} exts

tools/otagui/src/services/update_metadata_pb.js

@@ -1249,6 +1249,7 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
          * @property {number|Long|null} [dstLength] InstallOperation dstLength
          * @property {Uint8Array|null} [dataSha256Hash] InstallOperation dataSha256Hash
          * @property {Uint8Array|null} [srcSha256Hash] InstallOperation srcSha256Hash
+         * @property {Object.<string,number>|null} [xorMap] InstallOperation xorMap
          */
 
         /**
@@ -1262,6 +1263,7 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
         function InstallOperation(properties) {
             this.srcExtents = [];
             this.dstExtents = [];
+            this.xorMap = {};
             if (properties)
                 for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
                     if (properties[keys[i]] != null)
@@ -1340,6 +1342,14 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
          */
         InstallOperation.prototype.srcSha256Hash = $util.newBuffer([]);
 
+        /**
+         * InstallOperation xorMap.
+         * @member {Object.<string,number>} xorMap
+         * @memberof chromeos_update_engine.InstallOperation
+         * @instance
+         */
+        InstallOperation.prototype.xorMap = $util.emptyObject;
+
         /**
          * Creates a new InstallOperation instance using the specified properties.
          * @function create
@@ -1383,6 +1393,9 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 writer.uint32(/* id 8, wireType 2 =*/66).bytes(message.dataSha256Hash);
             if (message.srcSha256Hash != null && Object.hasOwnProperty.call(message, "srcSha256Hash"))
                 writer.uint32(/* id 9, wireType 2 =*/74).bytes(message.srcSha256Hash);
+            if (message.xorMap != null && Object.hasOwnProperty.call(message, "xorMap"))
+                for (let keys = Object.keys(message.xorMap), i = 0; i < keys.length; ++i)
+                    writer.uint32(/* id 10, wireType 2 =*/82).fork().uint32(/* id 1, wireType 0 =*/8).uint32(keys[i]).uint32(/* id 2, wireType 0 =*/16).uint32(message.xorMap[keys[i]]).ldelim();
             return writer;
         };
 
@@ -1413,7 +1426,7 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
         InstallOperation.decode = function decode(reader, length) {
             if (!(reader instanceof $Reader))
                 reader = $Reader.create(reader);
-            let end = length === undefined ? reader.len : reader.pos + length, message = new $root.chromeos_update_engine.InstallOperation();
+            let end = length === undefined ? reader.len : reader.pos + length, message = new $root.chromeos_update_engine.InstallOperation(), key, value;
             while (reader.pos < end) {
                 let tag = reader.uint32();
                 switch (tag >>> 3) {
@@ -1448,6 +1461,28 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 case 9:
                     message.srcSha256Hash = reader.bytes();
                     break;
+                case 10:
+                    if (message.xorMap === $util.emptyObject)
+                        message.xorMap = {};
+                    let end2 = reader.uint32() + reader.pos;
+                    key = 0;
+                    value = 0;
+                    while (reader.pos < end2) {
+                        let tag2 = reader.uint32();
+                        switch (tag2 >>> 3) {
+                        case 1:
+                            key = reader.uint32();
+                            break;
+                        case 2:
+                            value = reader.uint32();
+                            break;
+                        default:
+                            reader.skipType(tag2 & 7);
+                            break;
+                        }
+                    }
+                    message.xorMap[key] = value;
+                    break;
                 default:
                     reader.skipType(tag & 7);
                     break;
@@ -1537,6 +1572,17 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
             if (message.srcSha256Hash != null && message.hasOwnProperty("srcSha256Hash"))
                 if (!(message.srcSha256Hash && typeof message.srcSha256Hash.length === "number" || $util.isString(message.srcSha256Hash)))
                     return "srcSha256Hash: buffer expected";
+            if (message.xorMap != null && message.hasOwnProperty("xorMap")) {
+                if (!$util.isObject(message.xorMap))
+                    return "xorMap: object expected";
+                let key = Object.keys(message.xorMap);
+                for (let i = 0; i < key.length; ++i) {
+                    if (!$util.key32Re.test(key[i]))
+                        return "xorMap: integer key{k:uint32} expected";
+                    if (!$util.isInteger(message.xorMap[key[i]]))
+                        return "xorMap: integer{k:uint32} expected";
+                }
+            }
             return null;
         };
 
@@ -1664,6 +1710,13 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                     $util.base64.decode(object.srcSha256Hash, message.srcSha256Hash = $util.newBuffer($util.base64.length(object.srcSha256Hash)), 0);
                 else if (object.srcSha256Hash.length)
                     message.srcSha256Hash = object.srcSha256Hash;
+            if (object.xorMap) {
+                if (typeof object.xorMap !== "object")
+                    throw TypeError(".chromeos_update_engine.InstallOperation.xorMap: object expected");
+                message.xorMap = {};
+                for (let keys = Object.keys(object.xorMap), i = 0; i < keys.length; ++i)
+                    message.xorMap[keys[i]] = object.xorMap[keys[i]] >>> 0;
+            }
             return message;
         };
 
@@ -1684,6 +1737,8 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 object.srcExtents = [];
                 object.dstExtents = [];
             }
+            if (options.objects || options.defaults)
+                object.xorMap = {};
             if (options.defaults) {
                 object.type = options.enums === String ? "REPLACE" : 0;
                 if ($util.Long) {
@@ -1757,6 +1812,12 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 object.dataSha256Hash = options.bytes === String ? $util.base64.encode(message.dataSha256Hash, 0, message.dataSha256Hash.length) : options.bytes === Array ? Array.prototype.slice.call(message.dataSha256Hash) : message.dataSha256Hash;
             if (message.srcSha256Hash != null && message.hasOwnProperty("srcSha256Hash"))
                 object.srcSha256Hash = options.bytes === String ? $util.base64.encode(message.srcSha256Hash, 0, message.srcSha256Hash.length) : options.bytes === Array ? Array.prototype.slice.call(message.srcSha256Hash) : message.srcSha256Hash;
+            let keys2;
+            if (message.xorMap && (keys2 = Object.keys(message.xorMap)).length) {
+                object.xorMap = {};
+                for (let j = 0; j < keys2.length; ++j)
+                    object.xorMap[keys2[j]] = message.xorMap[keys2[j]];
+            }
             return object;
         };
 
@@ -1815,6 +1876,7 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
          * @property {chromeos_update_engine.CowMergeOperation.Type|null} [type] CowMergeOperation type
          * @property {chromeos_update_engine.IExtent|null} [srcExtent] CowMergeOperation srcExtent
          * @property {chromeos_update_engine.IExtent|null} [dstExtent] CowMergeOperation dstExtent
+         * @property {number|Long|null} [srcOffset] CowMergeOperation srcOffset
          */
 
         /**
@@ -1856,6 +1918,14 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
          */
         CowMergeOperation.prototype.dstExtent = null;
 
+        /**
+         * CowMergeOperation srcOffset.
+         * @member {number|Long} srcOffset
+         * @memberof chromeos_update_engine.CowMergeOperation
+         * @instance
+         */
+        CowMergeOperation.prototype.srcOffset = $util.Long ? $util.Long.fromBits(0,0,true) : 0;
+
         /**
          * Creates a new CowMergeOperation instance using the specified properties.
          * @function create
@@ -1886,6 +1956,8 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 $root.chromeos_update_engine.Extent.encode(message.srcExtent, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
             if (message.dstExtent != null && Object.hasOwnProperty.call(message, "dstExtent"))
                 $root.chromeos_update_engine.Extent.encode(message.dstExtent, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim();
+            if (message.srcOffset != null && Object.hasOwnProperty.call(message, "srcOffset"))
+                writer.uint32(/* id 4, wireType 0 =*/32).uint64(message.srcOffset);
             return writer;
         };
 
@@ -1929,6 +2001,9 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 case 3:
                     message.dstExtent = $root.chromeos_update_engine.Extent.decode(reader, reader.uint32());
                     break;
+                case 4:
+                    message.srcOffset = reader.uint64();
+                    break;
                 default:
                     reader.skipType(tag & 7);
                     break;
@@ -1969,6 +2044,8 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 default:
                     return "type: enum value expected";
                 case 0:
+                case 1:
+                case 2:
                     break;
                 }
             if (message.srcExtent != null && message.hasOwnProperty("srcExtent")) {
@@ -1981,6 +2058,9 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 if (error)
                     return "dstExtent." + error;
             }
+            if (message.srcOffset != null && message.hasOwnProperty("srcOffset"))
+                if (!$util.isInteger(message.srcOffset) && !(message.srcOffset && $util.isInteger(message.srcOffset.low) && $util.isInteger(message.srcOffset.high)))
+                    return "srcOffset: integer|Long expected";
             return null;
         };
 
@@ -2001,6 +2081,14 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
             case 0:
                 message.type = 0;
                 break;
+            case "COW_XOR":
+            case 1:
+                message.type = 1;
+                break;
+            case "COW_REPLACE":
+            case 2:
+                message.type = 2;
+                break;
             }
             if (object.srcExtent != null) {
                 if (typeof object.srcExtent !== "object")
@@ -2012,6 +2100,15 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                     throw TypeError(".chromeos_update_engine.CowMergeOperation.dstExtent: object expected");
                 message.dstExtent = $root.chromeos_update_engine.Extent.fromObject(object.dstExtent);
             }
+            if (object.srcOffset != null)
+                if ($util.Long)
+                    (message.srcOffset = $util.Long.fromValue(object.srcOffset)).unsigned = true;
+                else if (typeof object.srcOffset === "string")
+                    message.srcOffset = parseInt(object.srcOffset, 10);
+                else if (typeof object.srcOffset === "number")
+                    message.srcOffset = object.srcOffset;
+                else if (typeof object.srcOffset === "object")
+                    message.srcOffset = new $util.LongBits(object.srcOffset.low >>> 0, object.srcOffset.high >>> 0).toNumber(true);
             return message;
         };
 
@@ -2032,6 +2129,11 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 object.type = options.enums === String ? "COW_COPY" : 0;
                 object.srcExtent = null;
                 object.dstExtent = null;
+                if ($util.Long) {
+                    let long = new $util.Long(0, 0, true);
+                    object.srcOffset = options.longs === String ? long.toString() : options.longs === Number ? long.toNumber() : long;
+                } else
+                    object.srcOffset = options.longs === String ? "0" : 0;
             }
             if (message.type != null && message.hasOwnProperty("type"))
                 object.type = options.enums === String ? $root.chromeos_update_engine.CowMergeOperation.Type[message.type] : message.type;
@@ -2039,6 +2141,11 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
                 object.srcExtent = $root.chromeos_update_engine.Extent.toObject(message.srcExtent, options);
             if (message.dstExtent != null && message.hasOwnProperty("dstExtent"))
                 object.dstExtent = $root.chromeos_update_engine.Extent.toObject(message.dstExtent, options);
+            if (message.srcOffset != null && message.hasOwnProperty("srcOffset"))
+                if (typeof message.srcOffset === "number")
+                    object.srcOffset = options.longs === String ? String(message.srcOffset) : message.srcOffset;
+                else
+                    object.srcOffset = options.longs === String ? $util.Long.prototype.toString.call(message.srcOffset) : options.longs === Number ? new $util.LongBits(message.srcOffset.low >>> 0, message.srcOffset.high >>> 0).toNumber(true) : message.srcOffset;
             return object;
         };
 
@@ -2058,10 +2165,14 @@ export const chromeos_update_engine = $root.chromeos_update_engine = (() => {
          * @name chromeos_update_engine.CowMergeOperation.Type
          * @enum {number}
          * @property {number} COW_COPY=0 COW_COPY value
+         * @property {number} COW_XOR=1 COW_XOR value
+         * @property {number} COW_REPLACE=2 COW_REPLACE value
          */
         CowMergeOperation.Type = (function() {
             const valuesById = {}, values = Object.create(valuesById);
             values[valuesById[0] = "COW_COPY"] = 0;
+            values[valuesById[1] = "COW_XOR"] = 1;
+            values[valuesById[2] = "COW_REPLACE"] = 2;
             return values;
         })();