Merge "Add support for partial update and target library." am: ae475e51a6

Original change: https://android-review.googlesource.com/c/platform/development/+/1735315

Change-Id: I04aa387f24157431b1225ac797d2c1414c1c1db7

tools/otagui/.gitignore

@@ -25,3 +25,4 @@ pnpm-debug.log*
 *.njsproj
 *.sln
 *.sw?
+*.db

tools/otagui/.prettierrc.js

@@ -1,4 +1,5 @@
 module.exports = {
   singleQuote: true,
-  semi: false
+  semi: false,
+  useTabs: false
 }

tools/otagui/ota_interface.py

@@ -49,14 +49,10 @@ class ProcessesManagement:
     def get_status(self):
         return [self.get_status_by_ID(id=id) for id in self.get_keys()]
 
-    def get_list(self, dir):
-        files = os.listdir(dir)
-        return files
-
     def ota_generate(self, args, id=0):
         command = ['ota_from_target_files']
         # Check essential configuration is properly set
-        if not os.path.isfile('target/' + args['target']):
+        if not os.path.isfile(args['target']):
             raise FileNotFoundError
         if not args['output']:
             raise SyntaxError
@@ -65,14 +61,17 @@ class ProcessesManagement:
         command.append('-k')
         command.append(
             '../../../build/make/target/product/security/testkey')
-        if args['incremental']:
-            if not os.path.isfile('target/' + args['incremental']):
+        if args['isIncremental']:
+            if not os.path.isfile(args['incremental']):
                 raise FileNotFoundError
             command.append('-i')
-            command.append('target/' + args['incremental'])
-        command.append('target/' + args['target'])
+            command.append(args['incremental'])
+        if args['isPartial']:
+            command.append('--partial')
+            command.append(args['partial'])
+        command.append(args['target'])
         command.append(args['output'])
-
+        # Start a subprocess and collect the output
         stderr_pipes = pipes.Template()
         stdout_pipes = pipes.Template()
         ferr = stderr_pipes.open(os.path.join(

tools/otagui/src/components/BaseCheckbox.vue

@@ -3,6 +3,7 @@
     type="checkbox"
     :checked="modelValue"
     class="field"
+    v-bind="$attrs"
     @change="$emit('update:modelValue', $event.target.checked)"
   >
   <label v-if="label"> {{ label }} </label>

tools/otagui/src/components/FileSelect.vue

@@ -0,0 +1,39 @@
+<template>
+  <label v-if="label"> {{ label }} </label>
+  <select
+    :value="modelValue"
+    class="field"
+    v-bind="$attrs"
+    @change="$emit('update:modelValue', $event.target.value)"
+  >
+    <option
+      v-for="option in options"
+      :key="option.file_name"
+      :value="option.path"
+      :selected="option.path === modelValue"
+    >
+      {{ option.file_name }}
+    </option>
+  </select>
+</template>
+
+
+
+<script>
+export default {
+  props: {
+    label: {
+      type: String,
+      default: '',
+    },
+    modelValue: {
+      type: [String, Number],
+      default: ''
+    },
+    options: {
+      type: Array,
+      required: true
+    }
+  }
+}
+</script>

tools/otagui/src/components/PartialCheckbox.vue

@@ -0,0 +1,37 @@
+<template>
+  <ul v-bind="$attrs">
+    <li
+      v-for="label in labels"
+      :key="label"
+    >
+      <input
+        type="checkbox"
+        :value="label"
+        :checked="modelValue.get(label)"
+        @change="updateSelected($event.target.value)"
+      >
+      <label v-if="label"> {{ label }} </label>
+    </li>
+  </ul>
+</template>
+
+<script>
+export default {
+  props: {
+    labels: {
+      type: Array,
+      default: new Array(),
+    },
+    modelValue: {
+      type: Map,
+      default: new Map(),
+    },
+  },
+  methods: {
+    updateSelected(newSelect) {
+      this.modelValue.set(newSelect, !this.modelValue.get(newSelect))
+      this.$emit('update:modelValue', this.modelValue)
+    },
+  },
+}
+</script>

tools/otagui/src/views/SimpleForm.vue

@@ -3,16 +3,16 @@
     <form @submit.prevent="sendForm">
       <UploadFile @file-uploaded="fetchTargetList" />
       <br>
-      <BaseSelect
-        v-if="input.incrementalStatus"
+      <FileSelect
+        v-if="input.isIncremental"
         v-model="input.incremental"
         label="Select the source file"
-        :options="targetList"
+        :options="targetDetails"
       />
-      <BaseSelect
+      <FileSelect
         v-model="input.target"
         label="Select the target file"
-        :options="targetList"
+        :options="targetDetails"
       />
       <button
         type="button"
@@ -24,13 +24,26 @@
         <BaseCheckbox
           v-model="input.verbose"
           :label="'Verbose'"
-        />
-        &emsp;
+        /> &emsp;
         <BaseCheckbox
-          v-model="input.incrementalStatus"
+          v-model="input.isIncremental"
           :label="'Incremental'"
         />
       </div>
+      <div>
+        <BaseCheckbox
+          v-model="input.isPartial"
+          :label="'Partial'"
+        />
+        <PartialCheckbox
+          v-if="input.isPartial"
+          v-model="partitionInclude"
+          :labels="updatePartitions"
+        />
+        <div v-if="input.isPartial">
+          Partial list: {{ partitionList }}
+        </div>
+      </div>
       <br>
       <BaseInput
         v-model="input.extra"
@@ -42,14 +55,53 @@
       </button>
     </form>
   </div>
+  <div>
+    <ul>
+      <h4>Build Library</h4>
+      <strong>
+        Careful: Use a same filename will overwrite the original build.
+      </strong>
+      <br>
+      <button @click="updateBuildLib">
+        Refresh the build Library (use with cautions)
+      </button>
+      <li
+        v-for="targetDetail in targetDetails"
+        :key="targetDetail.file_name"
+      >
+        <div>
+          <h5>Build File Name: {{ targetDetail.file_name }}</h5>
+          Uploaded time: {{ formDate(targetDetail.time) }}
+          <br>
+          Build ID: {{ targetDetail.build_id }}
+          <br>
+          Build Version: {{ targetDetail.build_version }}
+          <br>
+          Build Flavor: {{ targetDetail.build_flavor }}
+          <br>
+          <button
+            :disabled="!input.isIncremental"
+            @click="selectIncremental(targetDetail.path)"
+          >
+            Select as Incremental File
+          </button>
+          &emsp;
+          <button @click="selectTarget(targetDetail.path)">
+            Select as Target File
+          </button>
+        </div>
+      </li>
+    </ul>
+  </div>
 </template>
 
 <script>
 import BaseInput from '@/components/BaseInput.vue'
 import BaseCheckbox from '@/components/BaseCheckbox.vue'
-import BaseSelect from '@/components/BaseSelect.vue'
+import FileSelect from '@/components/FileSelect.vue'
 import ApiService from '../services/ApiService.js'
 import UploadFile from '@/components/UploadFile.vue'
+import PartialCheckbox from '@/components/PartialCheckbox.vue'
 import { uuid } from 'vue-uuid'
 
 export default {
@@ -57,58 +109,76 @@ export default {
     BaseInput,
     BaseCheckbox,
     UploadFile,
-    BaseSelect,
+    FileSelect,
+    PartialCheckbox,
   },
   data() {
     return {
       id: 0,
-      input: {
-        verbose: false,
-        target: '',
-        output: 'output/',
-        incremental: '',
-        incrementalStatus: false,
-        extra: '',
-      },
+      input: {},
       inputs: [],
       response_message: '',
-      targetList: [],
+      targetDetails: [],
+      partitionInclude: new Map(),
     }
   },
   computed: {
-    updateOutput() {
-      return 'output/' + String(this.id) + '.zip'
+    updatePartitions() {
+      let target = this.targetDetails.filter(
+        (d) => d.path === this.input.target
+      )
+      return target[0].partitions
+    },
+    partitionList() {
+      let list = ''
+      for (let [key, value] of this.partitionInclude) {
+        if (value) {
+          list += key + ' '
+        }
+      }
+      return list
+    },
+  },
+  watch: {
+    partitionList: {
+      handler: function () {
+        this.input.partial = this.partitionList
+      },
     },
   },
   created() {
+    this.resetInput()
     this.fetchTargetList()
     this.updateUUID()
   },
   methods: {
-    sendForm(e) {
-      // console.log(this.input)
-      ApiService.postInput(this.input, this.id)
-        .then((Response) => {
-          this.response_message = Response.data
-          alert(this.response_message)
-        })
-        .catch((err) => {
-          this.response_message = 'Error! ' + err
-        })
+    resetInput() {
       this.input = {
         verbose: false,
         target: '',
         output: 'output/',
         incremental: '',
-        incrementalStatus: false,
+        isIncremental: false,
+        partial: '',
+        isPartial: false,
         extra: '',
       }
+    },
+    async sendForm(e) {
+      try {
+        let response = await ApiService.postInput(this.input, this.id)
+        this.response_message = response.data
+        alert(this.response_message)
+      } catch (err) {
+        console.log(err)
+      }
+      this.resetInput()
       this.updateUUID()
     },
     async fetchTargetList() {
       try {
-        let response = await ApiService.getFileList('/target')
-        this.targetList = response.data
+        let response = await ApiService.getFileList('')
+        this.targetDetails = response.data
       } catch (err) {
         console.log('Fetch Error', err)
       }
@@ -117,6 +187,36 @@ export default {
       this.id = uuid.v1()
       this.input.output += String(this.id) + '.zip'
     },
+    formDate(unixTime) {
+      let formTime = new Date(unixTime * 1000)
+      let date =
+        formTime.getFullYear() +
+        '-' +
+        (formTime.getMonth() + 1) +
+        '-' +
+        formTime.getDate()
+      let time =
+        formTime.getHours() +
+        ':' +
+        formTime.getMinutes() +
+        ':' +
+        formTime.getSeconds()
+      return date + ' ' + time
+    },
+    selectTarget(path) {
+      this.input.target = path
+    },
+    selectIncremental(path) {
+      this.input.incremental = path
+    },
+    async updateBuildLib() {
+      try {
+        let response = await ApiService.getFileList('/target')
+        this.targetDetails = response.data
+      } catch (err) {
+        console.log('Fetch Error', err)
+      }
+    },
   },
 }
 </script>

tools/otagui/target_lib.py

@@ -0,0 +1,158 @@
+from dataclasses import dataclass, asdict, field
+import sqlite3
+import time
+import logging
+import os
+import zipfile
+import re
+import json
+
+
+@dataclass
+class BuildInfo:
+    """
+    A class for Android build information.
+    """
+    file_name: str
+    path: str
+    time: int
+    build_id: str = ''
+    build_version: str = ''
+    build_flavor: str = ''
+    partitions: list[str] = field(default_factory=list)
+
+    def analyse_buildprop(self):
+        """
+        Analyse the build's version info and partitions included
+        Then write them into the build_info
+        """
+        def extract_info(pattern, lines):
+            # Try to match a regex in a list of string
+            line = list(filter(pattern.search, lines))[0]
+            if line:
+                return pattern.search(line).group(0)
+            else:
+                return ''
+
+        build = zipfile.ZipFile(self.path)
+        try:
+            with build.open('SYSTEM/build.prop', 'r') as build_prop:
+                raw_info = build_prop.readlines()
+                pattern_id = re.compile(b'(?<=ro\.build\.id\=).+')
+                pattern_version = re.compile(
+                    b'(?<=ro\.build\.version\.incremental\=).+')
+                pattern_flavor = re.compile(b'(?<=ro\.build\.flavor\=).+')
+                self.build_id = extract_info(
+                    pattern_id, raw_info).decode('utf-8')
+                self.build_version = extract_info(
+                    pattern_version, raw_info).decode('utf-8')
+                self.build_flavor = extract_info(
+                    pattern_flavor, raw_info).decode('utf-8')
+        except KeyError:
+            pass
+        try:
+            with build.open('META/ab_partitions.txt', 'r') as partition_info:
+                raw_info = partition_info.readlines()
+                for line in raw_info:
+                    self.partitions.append(line.decode('utf-8').rstrip())
+        except KeyError:
+            pass
+
+    def to_sql_form_dict(self):
+        sql_form_dict = asdict(self)
+        sql_form_dict['partitions'] = ','.join(sql_form_dict['partitions'])
+        return sql_form_dict
+
+    def to_dict(self):
+        return asdict(self)
+
+
+class TargetLib:
+    def __init__(self, path='ota_database.db'):
+        """
+        Create a build table if not existing
+        """
+        self.path = path
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+                CREATE TABLE if not exists Builds (
+                FileName TEXT,
+                UploadTime INTEGER,
+                Path TEXT,
+                BuildID TEXT,
+                BuildVersion TEXT,
+                BuildFlavor TEXT,
+                Partitions TEXT
+            )
+            """)
+
+    def new_build(self, filename, path):
+        """
+        Insert a new build into the database
+        Args:
+            filename: the name of the file
+            path: the relative path of the file
+        """
+        build_info = BuildInfo(filename, path, int(time.time()))
+        build_info.analyse_buildprop()
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+            SELECT * FROM Builds WHERE FileName=:file_name and Path=:path
+            """, build_info.to_sql_form_dict())
+            if cursor.fetchall():
+                cursor.execute("""
+                DELETE FROM Builds WHERE FileName=:file_name and Path=:path
+                """, build_info.to_sql_form_dict())
+            cursor.execute("""
+            INSERT INTO Builds (FileName, UploadTime, Path, BuildID, BuildVersion, BuildFlavor, Partitions)
+            VALUES (:file_name, :time, :path, :build_id, :build_version, :build_flavor, :partitions)
+            """, build_info.to_sql_form_dict())
+
+    def new_build_from_dir(self, path):
+        """
+        Update the database using files under a directory
+        Args:
+            path: a directory
+        """
+        if os.path.isdir(path):
+            builds_name = os.listdir(path)
+            for build_name in builds_name:
+                self.new_build(build_name, os.path.join(path, build_name))
+        elif os.path.isfile(path):
+            self.new_build(os.path.split(path)[-1], path)
+        return self.get_builds()
+
+    def sql_to_buildinfo(self, row):
+        build_info = BuildInfo(*row[:6], row[6].split(','))
+        return build_info
+
+    def get_builds(self):
+        """
+        Get a list of builds in the database
+        Return:
+            A list of build_info, each of which is an object:
+            (FileName, UploadTime, Path, Build ID, Build Version, Build Flavor, Partitions)
+        """
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+            SELECT FileName, Path, UploadTime, BuildID, BuildVersion, BuildFlavor, Partitions
+            FROM Builds""")
+            return list(map(self.sql_to_buildinfo, cursor.fetchall()))
+
+    def get_builds_by_path(self, path):
+        """
+        Get a build in the database by its path
+        Return:
+            A build_info, which is an object:
+            (FileName, UploadTime, Path, Build ID, Build Version, Build Flavor, Partitions)
+        """
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+            SELECT FileName, Path, UploadTime, BuildID, BuildVersion, BuildFlavor, Partitions
+            WHERE Path==(?)
+            """, (path, ))
+        return self.sql_to_buildinfo(cursor.fetchone())

tools/otagui/web_server.py

@@ -9,12 +9,17 @@ API::
   GET /check : check the status of all jobs
   GET /check/<id> : check the status of the job with <id>
   GET /file : fetch the target file list
+  GET /file/<path> : Add build file(s) in <path>, and return the target file list
   GET /download/<id> : download the ota package with <id>
   POST /run/<id> : submit a job with <id>,
                  arguments set in a json uploaded together
   POST /file/<filename> : upload a target file
   [TODO] POST /cancel/<id> : cancel a job with <id>
 
+TODO:
+  - Avoid unintentionally path leakage
+  - Avoid overwriting build when uploading build with same file name
+
 Other GET request will be redirected to the static request under 'dist' directory
 """
 
@@ -22,12 +27,14 @@ from http.server import BaseHTTPRequestHandler, SimpleHTTPRequestHandler, HTTPSe
 from socketserver import ThreadingMixIn
 from threading import Lock
 from ota_interface import ProcessesManagement
+from target_lib import TargetLib
 import logging
 import json
 import pipes
 import cgi
 import subprocess
 import os
+import sys
 
 LOCAL_ADDRESS = '0.0.0.0'
 
@@ -77,10 +84,14 @@ class RequestHandler(CORSSimpleHTTPHandler):
             )
             return
         elif self.path.startswith('/file'):
-            file_list = jobs.get_list(self.path[6:])
+            if self.path == '/file' or self.path == '/file/':
+                file_list = target_lib.get_builds()
+            else:
+                file_list = target_lib.new_build_from_dir(self.path[6:])
+            builds_info = [build.to_dict() for build in file_list]
             self._set_response(type='application/json')
             self.wfile.write(
-                json.dumps(file_list).encode()
+                json.dumps(builds_info).encode()
             )
             logging.info(
                 "GET request:\nPath:%s\nHeaders:\n%s\nBody:\n%s\n",
@@ -133,6 +144,7 @@ class RequestHandler(CORSSimpleHTTPHandler):
                 file_length -= len(self.rfile.readline())
                 file_length -= len(self.rfile.readline())
                 output_file.write(self.rfile.read(file_length))
+                target_lib.new_build(self.path[6:], file_name)
             self._set_response(code=201)
             self.wfile.write(
                 "File received, saved into {}".format(
@@ -164,8 +176,16 @@ def run_server(SeverClass=ThreadedHTTPServer, HandlerClass=RequestHandler, port=
 if __name__ == '__main__':
     from sys import argv
     print(argv)
+    if not os.path.isdir('target'):
+        os.mkdir('target', 755)
+    if not os.path.isdir('output'):
+        os.mkdir('output', 755)
+    target_lib = TargetLib()
     jobs = ProcessesManagement()
-    if len(argv) == 2:
-        run_server(port=int(argv[1]))
-    else:
-        run_server()
+    try:
+        if len(argv) == 2:
+            run_server(port=int(argv[1]))
+        else:
+            run_server()
+    except KeyboardInterrupt:
+        sys.exit(0)