From d68935075640e69c07964b09e2f974942d94c669 Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Tue, 27 Jul 2021 20:11:21 -0400 Subject: [PATCH] Support docker for ota generator Test: docker run -it -p 8000:8000 -v target:/app/target -v output:/app/output zhangxp1998/test:latest Change-Id: Ife050e6a8c85f2d40aff82a752b19f27c2ab23c5 --- tools/otagui/.dockerignore | 2 ++ tools/otagui/.env.development | 1 + tools/otagui/.env.production | 1 + tools/otagui/.gitignore | 2 ++ tools/otagui/Dockerfile | 20 ++++++++++++++++++++ tools/otagui/README.md | 5 +++++ tools/otagui/ota_interface.py | 10 +++++----- tools/otagui/src/services/ApiService.js | 11 +++++++++-- tools/otagui/src/views/JobDetails.vue | 2 +- tools/otagui/target_lib.py | 2 +- tools/otagui/web_server.py | 25 +++++++++++++++++++++---- 11 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 tools/otagui/.dockerignore create mode 100644 tools/otagui/.env.development create mode 100644 tools/otagui/.env.production create mode 100644 tools/otagui/Dockerfile diff --git a/tools/otagui/.dockerignore b/tools/otagui/.dockerignore new file mode 100644 index 000000000..e3f4a471c --- /dev/null +++ b/tools/otagui/.dockerignore @@ -0,0 +1,2 @@ +node_modules/ +Dockerfile \ No newline at end of file diff --git a/tools/otagui/.env.development b/tools/otagui/.env.development new file mode 100644 index 000000000..c0d665211 --- /dev/null +++ b/tools/otagui/.env.development @@ -0,0 +1 @@ +NODE_ENV=development diff --git a/tools/otagui/.env.production b/tools/otagui/.env.production new file mode 100644 index 000000000..cbde1ccac --- /dev/null +++ b/tools/otagui/.env.production @@ -0,0 +1 @@ +NODE_ENV=production diff --git a/tools/otagui/.gitignore b/tools/otagui/.gitignore index b81e39b9a..323330f74 100644 --- a/tools/otagui/.gitignore +++ b/tools/otagui/.gitignore @@ -5,6 +5,8 @@ node_modules /output stderr* stdout* +yarn.lock +otatools.zip # local env files diff --git a/tools/otagui/Dockerfile b/tools/otagui/Dockerfile new file mode 100644 index 000000000..5be13ef86 --- /dev/null +++ b/tools/otagui/Dockerfile @@ -0,0 +1,20 @@ +# build stage +FROM node:lts-alpine as build-stage +WORKDIR /app +COPY package*.json ./ +RUN npm install +COPY . . +RUN npm run build + +# production stage +FROM ubuntu:20.04 as production-stage +RUN apt-get update && apt-get --no-install-recommends install -y python3.9 unzip xxd cgpt unzip openjdk-16-jre-headless zip less + +WORKDIR /app +VOLUME [ "/app/target", "/app/output"] +COPY otatools.zip . +COPY --from=build-stage /app/dist ./dist +COPY --from=build-stage /app/*.py . + +EXPOSE 8000 +CMD ["python3.9", "web_server.py"] \ No newline at end of file diff --git a/tools/otagui/README.md b/tools/otagui/README.md index f4ec98729..1fb541ad6 100644 --- a/tools/otagui/README.md +++ b/tools/otagui/README.md @@ -27,3 +27,8 @@ Finally, run the python http-server and vue.js server: python3 web_server.py & npm run serve ``` +### Run with Docker + +1. Build the image `docker build -t zhangxp1998/test .` + +2. Run: `docker run -it -p 8000:8000 -v target:/app/target -v output:/app/output zhangxp1998/test:latest` diff --git a/tools/otagui/ota_interface.py b/tools/otagui/ota_interface.py index a0ca06542..9cdd90130 100644 --- a/tools/otagui/ota_interface.py +++ b/tools/otagui/ota_interface.py @@ -89,7 +89,7 @@ class ProcessesManagement: A class manage the ota generate process """ - def __init__(self, path='ota_database.db'): + def __init__(self, path='output/ota_database.db'): """ create a table if not exist """ @@ -156,9 +156,9 @@ class ProcessesManagement: 'output', 'stdout.'+str(id)), 'w') try: proc = subprocess.Popen( - command, stderr=ferr, stdout=fout) - except FileNotFoundError: - logging.error('ota_from_target_files is not set properly') + command, stderr=ferr, stdout=fout, shell=False) + except FileNotFoundError as e: + logging.error('ota_from_target_files is not set properly %s', e) self.update_status(id, 'Error', int(time.time())) return exit_code = proc.wait() @@ -182,7 +182,7 @@ class ProcessesManagement: command += args['extra'].split(' ') command.append('-k') command.append( - '../../../build/make/target/product/security/testkey') + 'build/make/target/product/security/testkey') if args['isIncremental']: if not os.path.isfile(args['incremental']): raise FileNotFoundError diff --git a/tools/otagui/src/services/ApiService.js b/tools/otagui/src/services/ApiService.js index 3a328d3f4..b79abf68b 100644 --- a/tools/otagui/src/services/ApiService.js +++ b/tools/otagui/src/services/ApiService.js @@ -1,15 +1,22 @@ import axios from 'axios' +const baseURL = process.env.NODE_ENV === 'production' ? '' : 'http://localhost:8000'; + +console.log(`Build mode: ${process.env.NODE_ENV}, API base url ${baseURL}`); + const apiClient = axios.create({ - baseURL: 'http://localhost:8000', + baseURL, withCredentials: false, headers: { Accept: 'application/json', 'Content-Type': 'application/json' } -}) +}); export default { + getDownloadURLForJob(job) { + return `${baseURL}/download/${job.output}`; + }, getJobs() { return apiClient.get("/check") }, diff --git a/tools/otagui/src/views/JobDetails.vue b/tools/otagui/src/views/JobDetails.vue index 11d426544..ad1967549 100644 --- a/tools/otagui/src/views/JobDetails.vue +++ b/tools/otagui/src/views/JobDetails.vue @@ -63,7 +63,7 @@ export default { }, computed: { download() { - return 'http://localhost:8000/download/' + this.job.output + return ApiService.getDownloadURLForJob(this.job); }, }, created() { diff --git a/tools/otagui/target_lib.py b/tools/otagui/target_lib.py index 02d62abb9..294ace2dd 100644 --- a/tools/otagui/target_lib.py +++ b/tools/otagui/target_lib.py @@ -71,7 +71,7 @@ class TargetLib: """ A class that manages the builds in database. """ - def __init__(self, path='ota_database.db'): + def __init__(self, path='target/ota_database.db'): """ Create a build table if not existing """ diff --git a/tools/otagui/web_server.py b/tools/otagui/web_server.py index e4ddb6188..00595e978 100644 --- a/tools/otagui/web_server.py +++ b/tools/otagui/web_server.py @@ -30,11 +30,11 @@ from ota_interface import ProcessesManagement from target_lib import TargetLib import logging import json -import pipes import cgi -import subprocess import os +import stat import sys +import zipfile LOCAL_ADDRESS = '0.0.0.0' @@ -142,7 +142,10 @@ class RequestHandler(CORSSimpleHTTPHandler): file_length -= len(self.rfile.readline()) file_length -= len(self.rfile.readline()) file_length -= len(self.rfile.readline()) - output_file.write(self.rfile.read(file_length)) + BUFFER_SIZE = 1024*1024 + for offset in range(0, file_length, BUFFER_SIZE): + chunk = self.rfile.read(min(file_length-offset, BUFFER_SIZE)) + output_file.write(chunk) target_lib.new_build(self.path[6:], file_name) self._set_response(code=201) self.wfile.write( @@ -158,7 +161,6 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): def run_server(SeverClass=ThreadedHTTPServer, HandlerClass=RequestHandler, port=8000): - logging.basicConfig(level=logging.DEBUG) server_address = (LOCAL_ADDRESS, port) server_instance = SeverClass(server_address, HandlerClass) try: @@ -169,12 +171,27 @@ def run_server(SeverClass=ThreadedHTTPServer, HandlerClass=RequestHandler, port= except KeyboardInterrupt: pass server_instance.server_close() + logging.basicConfig(level=logging.DEBUG) logging.info('Server has been turned off.') if __name__ == '__main__': from sys import argv + logging.basicConfig(level=logging.DEBUG) print(argv) + if os.path.exists("otatools.zip"): + logging.info("Found otatools.zip, extracting...") + EXTRACT_DIR = "./" + os.makedirs(EXTRACT_DIR, exist_ok=True) + with zipfile.ZipFile("otatools.zip", "r") as zfp: + zfp.extractall(EXTRACT_DIR) + # mark all binaries executable by owner + bin_dir = os.path.join(EXTRACT_DIR, "bin") + for filename in os.listdir(bin_dir): + os.chmod(os.path.join(bin_dir, filename), stat.S_IRWXU) + os.environ["PATH"] = os.path.join(EXTRACT_DIR, "bin") + ":" + os.environ["PATH"] + logging.info("Extracted otatools to {}".format(EXTRACT_DIR)) + logging.info("PATH: %s", os.environ["PATH"]) if not os.path.isdir('target'): os.mkdir('target', 755) if not os.path.isdir('output'):