Update import automation GKE deployment scripts and documentation

import-automation/executor/Dockerfile

@@ -13,16 +13,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM gcr.io/google-appengine/python
 
-RUN virtualenv /env -p python3.7
+FROM python:3.11.4
 
-ENV VIRTUAL_ENV /env
-ENV PATH /env/bin:$PATH
+RUN apt upgrade
+WORKDIR /workspace
 
-ADD requirements.txt /app/requirements.txt
-RUN pip install -r /app/requirements.txt
+ADD requirements.txt /workspace/requirements.txt
+RUN pip install -r /workspace/requirements.txt
 
-ADD . /app
+COPY app/. /workspace/app/
 
 CMD gunicorn --timeout 1800 --workers 5 -b :$PORT app.main:FLASK_APP

import-automation/executor/gke/README.md

@@ -1,18 +1,53 @@
-Deploy to GKE
+# Deploy Executor to GKE
 
-Import-automation executor can also be deployed to gke. After committing new changes, please do the following to deploy.
+Import-automation executor can be deployed to Google Kubernetes Engine
+(GKE).
 
-1. From repo root, run: `cd import-automation/executor`
+## (One Time) Setup IAP
 
-2. build and push image: `./gke/push_image.sh`
+Import Automation tool use [Identity-Aware Proxy
+(IAP)](https://cloud.google.com/iap) to guard access. Non public instances are
+guarded by IAP.
 
-3. Replace image tag in deployment.yaml with the value of `git rev-parse --short=7 HEAD`.
+### Configure the OAuth consent screen
 
-4. Update GKE with the following.
+Follow
+[documentation](https://cloud.google.com/iap/docs/enabling-kubernetes-howto#oauth-configure).
+
+### Create OAuth credential
+
+Follow
+[documentation](https://cloud.google.com/iap/docs/enabling-kubernetes-howto#oauth-credentials).
+
+### Setup IAP access
+
+Follow
+[documentation](https://cloud.google.com/iap/docs/enabling-kubernetes-howto#iap-access).
+
+## (One Time) Setup GKE
+
+1. Update OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in "gke/configure_gke.sh".
+2. Run `./gke/configure_gke.sh`.
+
+## Deployment
+
+After committing new changes, please do the following to deploy.
+
+1. Make sure the local repo is clean without pending changes.
+
+2. From repo root, run: `cd import-automation/executor`.
+
+3. build and push image: `./gke/push_image.sh`. This build a new docker image
+   with tag `prod` and the current git commit hash.
+
+4. Update GKE with the following command (replace GCP_PROJECT with actual
+   project id):
 
 ```sh
+export PROJECT_ID=<GCP_PROJECT>
+
 gcloud container clusters get-credentials datacommons-us-central1 \
-  --region us-central1 --project datcom-website-dev
+  --region us-central1 --project $PROJECT_ID
 
 kubectl apply -f gke/deployment.yaml
 ```

import-automation/executor/gke/cloudbuild.push_image.yaml

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 steps:
   - id: push-import-automation-executor
     name: gcr.io/cloud-builders/docker
@@ -19,8 +20,13 @@ steps:
       - -c
       - |
         set -e
-        DOCKER_BUILDKIT=1 docker build -f Dockerfile --tag gcr.io/$_PROJECT_ID/datacommons-import-automation-executor:${_TAG} .
-        docker push gcr.io/$_PROJECT_ID/datacommons-import-automation-executor:${_TAG}
+        DOCKER_BUILDKIT=1
+        docker build -f Dockerfile \
+          --tag gcr.io/datcom-ci/datacommons-import-automation-executor:${_TAG} \
+          --tag gcr.io/datcom-ci/datacommons-import-automation-executor:prod \
+          .
+        docker push gcr.io/datcom-ci/datacommons-import-automation-executor:${_TAG}
+        docker push gcr.io/datcom-ci/datacommons-import-automation-executor:prod
 
 timeout: 600s
 options:

import-automation/executor/gke/configure_gke.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,11 +13,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#!/bin/bash
-gcloud container clusters get-credentials \
-  datacommons-us-central1 \
+
+PROJECT_ID=datcom-data
+
+gcloud config set project $PROJECT_ID
+
+# Create GKE cluster
+gcloud container clusters create datacommons-us-central1 \
+  --num-nodes=3 \
+  --region=us-central1 \
+  --machine-type=e2-highmem-4 \
+  --enable-ip-alias \
+  --workload-pool=$PROJECT_ID.svc.id.goog \
+  --scopes=https://www.googleapis.com/auth/trace.append
+
+gcloud container clusters get-credentials datacommons-us-central1 \
   --region us-central1 \
-  --project datcom-website-dev
+  --project $PROJECT_ID
 
 # Create namespace if it does not exist.
 kubectl create namespace import-automation \
@@ -27,16 +41,16 @@ kubectl create serviceaccount --namespace import-automation import-automation-ks
   --dry-run=client -o yaml | kubectl apply -f -
 
 gcloud iam service-accounts add-iam-policy-binding \
-  --project datcom-website-dev \
+  --project $PROJECT_ID \
   --role roles/iam.workloadIdentityUser \
-  --member "serviceAccount:datcom-website-dev.svc.id.goog[import-automation/import-automation-ksa]" \
-  datcom-website-dev@appspot.gserviceaccount.com
+  --member "serviceAccount:$PROJECT_ID.svc.id.goog[import-automation/import-automation-ksa]" \
+  $PROJECT_ID@appspot.gserviceaccount.com
 
 kubectl annotate serviceaccount \
   --namespace import-automation \
   --overwrite \
   import-automation-ksa \
-  iam.gke.io/gcp-service-account=datcom-website-dev@appspot.gserviceaccount.com
+  iam.gke.io/gcp-service-account=$PROJECT_ID@appspot.gserviceaccount.com
 
 # Set the oauth env vars before running the script
 # export OAUTH_CLIENT_ID=<fill>
@@ -46,7 +60,6 @@ kubectl -n import-automation create secret generic import-automation-iap-secret
   --from-literal=client_secret=$OAUTH_CLIENT_SECRET
 
 # Also set what identity will cloud scheduler call as by running:
-# export CLOUD_SCHEDULER_CALLER_SA=<fill>
 kubectl -n import-automation create configmap cluster-oauth-configmap \
-  --from-literal=cloud-scheduler-caller-sa=$CLOUD_SCHEDULER_CALLER_SA \
+  --from-literal=cloud-scheduler-caller-sa=$[email protected] \
   --from-literal=cloud-scheduler-caller-oauth-audience=$OAUTH_CLIENT_ID

import-automation/executor/gke/deployment.yaml

@@ -28,13 +28,13 @@ spec:
   rules:
     - http:
         paths:
-        - path: /*
-          pathType: ImplementationSpecific
-          backend:
-            service:
-              name: import-automation-executor-service
-              port:
-                number: 8080
+          - path: /*
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: import-automation-executor-service
+                port:
+                  number: 8080
 ---
 apiVersion: cloud.google.com/v1
 kind: BackendConfig
@@ -82,7 +82,7 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app:  import-automation-executor-app
+      app: import-automation-executor-app
   template:
     metadata:
       labels:
@@ -128,4 +128,3 @@ spec:
               path: /healthz
               port: 8080
       serviceAccountName: import-automation-ksa
-

import-automation/executor/gke/push_image.sh

@@ -15,14 +15,8 @@
 # limitations under the License.
 set -e
 
-PROJECT_ID=$1
-
-if [[ $PROJECT_ID == "" ]]; then
-  PROJECT_ID=datcom-ci
-fi
-
 gcloud builds submit . \
   --async \
-  --project=$PROJECT_ID \
+  --project=datcom-ci \
   --config=gke/cloudbuild.push_image.yaml \
-  --substitutions=_TAG="prod",_PROJECT_ID=$PROJECT_ID
+  --substitutions=_TAG=$(git rev-parse --short=7 HEAD)