From e294207af6b13e694d1acf0bb9e975ecfaaf973d Mon Sep 17 00:00:00 2001 From: Scott Lobdell Date: Tue, 24 Apr 2018 11:27:23 -0700 Subject: [PATCH] Update project scripts for correct bootstrapping and GCE deployment Test: Started from an empty environment and cleaned up bootstrap commands again Change-Id: I455fdb883240aed952b6a59a38f247bc5a1e8095 --- tools/repo_diff/service/repodiff/Makefile | 44 +++++++++++++------ tools/repo_diff/service/repodiff/config.json | 4 +- .../repodiff/remote_scripts/gce_startup.sh | 23 ++++++++++ .../repodiff/remote_scripts/kill_self.sh | 12 +++++ .../tools/clear_service_account_keys.py | 38 ++++++++++++++++ 5 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 tools/repo_diff/service/repodiff/remote_scripts/gce_startup.sh create mode 100755 tools/repo_diff/service/repodiff/remote_scripts/kill_self.sh create mode 100755 tools/repo_diff/service/repodiff/tools/clear_service_account_keys.py diff --git a/tools/repo_diff/service/repodiff/Makefile b/tools/repo_diff/service/repodiff/Makefile index 6311d5681..866c84fa9 100644 --- a/tools/repo_diff/service/repodiff/Makefile +++ b/tools/repo_diff/service/repodiff/Makefile @@ -14,15 +14,22 @@ DOCKER_CANONICAL_ID=$(DOCKER_CONTAINER_REGISTRY)/$(GOOGLE_PROJECT_ID)/$(DOCKER_I PORT_HTTP="80" GCE_ZONE="us-west1-a" +GCE_IMAGE_PROJECT="ubuntu-os-cloud" +GCE_IMAGE_FAMILY="ubuntu-1604-lts" + +TMP_CREDENTIAL_FNAME=service_account_credentials.json # https://cloud.google.com/compute/docs/machine-types -GCE_MACHINE_TYPE="n1-standard-16" +GCE_MACHINE_TYPE="n1-standard-64" PROJECT_NAME="auto-diff-android-branches" REMOTE_MACHINE_NAME=mithalop5 FIREWALL_NAME=public-http-access DISK_SIZE=500GB RUN_COMMAND_REMOTE=gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) "$(REMOTE_MACHINE_NAME)" --command +SCP_TO_HOST=gcloud compute --project $(PROJECT_NAME) scp --zone $(GCE_ZONE) +SERVICE_ACCOUNT_ID=repo-reader +SERVICE_ACCOUNT=$(SERVICE_ACCOUNT_ID)@$(PROJECT_NAME).iam.gserviceaccount.com ifeq ($(ROLE),prod) GCP_DB_USER=$(GCP_DB_USER_PROD) @@ -42,14 +49,13 @@ endif bootstrap: mkdir -p $(GOPATH)/src ./tools/setup_go_path_symlink.sh - cd $(GOPATH)/src/repodiff # include $GOPATH/bin as part of system path grep -q -F 'export PATH=$$PATH:$$GOPATH/bin' ~/.bashrc || echo 'export PATH=$$PATH:$$GOPATH/bin' >> ~/.bashrc source ~/.bashrc - go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy - go get github.com/golang/dep/cmd/dep - dep ensure - go build; + cd $(GOPATH)/src/repodiff; go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy; \ + go get github.com/golang/dep/cmd/dep; \ + dep ensure; \ + go build run: go build; @@ -99,23 +105,35 @@ ssh: gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) $(REMOTE_MACHINE_NAME) deploy: + gcloud config set project $(PROJECT_NAME) @echo "Starting docker image build" make build_container_image @echo "Creating machine if it doesn't already exist" - gcloud beta compute instances create-with-container $(REMOTE_MACHINE_NAME) \ - --container-image $(DOCKER_CANONICAL_ID) \ - --tags $(DOCKER_TAG_NAME) \ + gcloud compute instances create $(REMOTE_MACHINE_NAME) \ --machine-type $(GCE_MACHINE_TYPE) \ --boot-disk-size $(DISK_SIZE) \ - --scopes https://www.googleapis.com/auth/source.read_only \ + --scopes https://www.googleapis.com/auth/source.read_only,https://www.googleapis.com/auth/compute \ --zone $(GCE_ZONE) \ + --local-ssd interface=nvme \ + --metadata-from-file startup-script=remote_scripts/gce_startup.sh \ + --metadata AUTHOR=$(USER),SERVICE_ACCOUNT=$(SERVICE_ACCOUNT),GOOGLE_PROJECT_ID=$(GOOGLE_PROJECT_ID) \ + --image-project $(GCE_IMAGE_PROJECT) \ + --image-family $(GCE_IMAGE_FAMILY) \ + --min-cpu-platform skylake \ + --service-account $(SERVICE_ACCOUNT) \ 2>/dev/null || true @echo "Hackily waiting a bit for instance to start up" - @sleep 10 + @sleep 30 + ./tools/clear_service_account_keys.py $(SERVICE_ACCOUNT) + gcloud iam service-accounts keys create $(TMP_CREDENTIAL_FNAME) --iam-account $(SERVICE_ACCOUNT) + $(RUN_COMMAND_REMOTE) 'mkdir -p /tmp/scripts' + $(SCP_TO_HOST) remote_scripts/* "$(REMOTE_MACHINE_NAME)":/tmp/scripts/ + $(SCP_TO_HOST) $(TMP_CREDENTIAL_FNAME) "$(REMOTE_MACHINE_NAME)":/tmp/ + rm $(TMP_CREDENTIAL_FNAME) @echo "Stopping all existing docker images" $(RUN_COMMAND_REMOTE) 'docker stop $$(docker ps -a -q)' 2>/dev/null || true docker image save $(DOCKER_CANONICAL_ID) -o transferrable_docker_image.tar \ - && gcloud compute --project $(PROJECT_NAME) scp --zone $(GCE_ZONE) transferrable_docker_image.tar "$(REMOTE_MACHINE_NAME)":~/transferred_docker_image.tar \ + && $(SCP_TO_HOST) transferrable_docker_image.tar "$(REMOTE_MACHINE_NAME)":~/transferred_docker_image.tar \ && $(RUN_COMMAND_REMOTE) 'docker load -i transferred_docker_image.tar' \ && $(RUN_COMMAND_REMOTE) 'docker run -d --rm -p $(DOCKER_TARGET_PORT):$(DOCKER_TARGET_PORT) $(DOCKER_CANONICAL_ID)' \ && gcloud compute firewall-rules create $(FIREWALL_NAME) --allow tcp:$(DOCKER_TARGET_PORT) 2>/dev/null || true \ @@ -127,7 +145,7 @@ output_instance_url: @echo "Monitor progress at http://"$(shell (gcloud compute instances list | grep $(REMOTE_MACHINE_NAME) | awk -F ' ' '{print $$5}')):$(DOCKER_TARGET_PORT)/health destroy: - gcloud compute instances delete $(REMOTE_MACHINE_NAME) + gcloud compute instances delete $(REMOTE_MACHINE_NAME) --zone $(GCE_ZONE) --quiet ############## DOCKER DEPLOYMENT build_container_image: diff --git a/tools/repo_diff/service/repodiff/config.json b/tools/repo_diff/service/repodiff/config.json index 1c8a6d6d7..d4c91dd7b 100644 --- a/tools/repo_diff/service/repodiff/config.json +++ b/tools/repo_diff/service/repodiff/config.json @@ -1,6 +1,6 @@ { - "android_project_dir": "/ssd850/tmp/development/tools/repo_diff/", - "output_directory": "/ssd850/slobdell_projects/app_output", + "android_project_dir": "/app/pytools/", + "output_directory": "/app/output", "diff_script": "repo_diff_android.py", "common_upstream": { "url": "https://keystone-qcom.googlesource.com/platform/manifest", diff --git a/tools/repo_diff/service/repodiff/remote_scripts/gce_startup.sh b/tools/repo_diff/service/repodiff/remote_scripts/gce_startup.sh new file mode 100644 index 000000000..9786faecf --- /dev/null +++ b/tools/repo_diff/service/repodiff/remote_scripts/gce_startup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# mount an SSD for fast repo syncing +mkfs.ext4 -F /dev/nvme0n1 +mkdir /ssd +mount /dev/nvme0n1 /ssd +chmod a+w /ssd + +# configure Docker to run on the SSD +mkdir -p /ssd/docker +mkdir -p /etc/docker +echo "{\"graph\": \"/ssd/docker\"}" > /etc/docker/daemon.json + +# install Docker +apt-get update +apt-get -qq -y --force-yes install docker.io +author=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/AUTHOR" -H "Metadata-Flavor: Google") +usermod -a -G docker $author + +# authenticate to Google Cloud as service account +serviceAccount=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/SERVICE_ACCOUNT" -H "Metadata-Flavor: Google") +googleProjectID=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/GOOGLE_PROJECT_ID" -H "Metadata-Flavor: Google") +gcloud projects add-iam-policy-binding $googleProjectID --member serviceAccount:$serviceAccount --role roles/compute.instanceAdmin.v1 diff --git a/tools/repo_diff/service/repodiff/remote_scripts/kill_self.sh b/tools/repo_diff/service/repodiff/remote_scripts/kill_self.sh new file mode 100755 index 000000000..99d4b339d --- /dev/null +++ b/tools/repo_diff/service/repodiff/remote_scripts/kill_self.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +zoneMetadata=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor:Google") +# Split on / and get the 4th element to get the actual zone name +IFS=$'/' +zoneMetadataSplit=($zoneMetadata) +zone="${zoneMetadataSplit[3]}" + +echo $(hostname) +echo $zone + +gcloud compute instances delete $(hostname) --zone=$zone --quiet diff --git a/tools/repo_diff/service/repodiff/tools/clear_service_account_keys.py b/tools/repo_diff/service/repodiff/tools/clear_service_account_keys.py new file mode 100755 index 000000000..5307a741f --- /dev/null +++ b/tools/repo_diff/service/repodiff/tools/clear_service_account_keys.py @@ -0,0 +1,38 @@ +#!/usr/bin/python + +import commands +import sys + + +def run_command(command): + return_code, output = commands.getstatusoutput(command) + if return_code != 0: + raise ValueError("Failed to execute command: %s" % command) + return output + + +def list_key_ids_for_service_account(service_account): + return parse_list_key_output( + run_command("gcloud iam service-accounts keys list --iam-account %s" % service_account) + ) + +def parse_list_key_output(output): + for line in [l for l in output.splitlines() if l][1:-1]: + key_id, created_at, expires_at = line.split() + yield key_id + + +def delete_keys(key_ids, service_account): + for key_id in key_ids: + run_command( + "gcloud iam service-accounts keys delete %s --iam-account %s --quiet" % (key_id, service_account), + ) + print "Deleted key %s" % key_id + + +if __name__ == "__main__": + service_account = sys.argv[1] + delete_keys( + list_key_ids_for_service_account(service_account), + service_account, + )