Import automation: Setup script updates

import-automation/executor/Dockerfile

@@ -16,7 +16,10 @@
 
 FROM python:3.11.4
 
-RUN apt upgrade
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /workspace
 
 ADD requirements.txt /workspace/requirements.txt

import-automation/executor/app/configs.py

@@ -34,7 +34,7 @@ class ExecutorConfig:
 
     # ID of the Google Cloud project that hosts the executor. The project
     # needs to enable App Engine and Cloud Scheduler.
-    gcp_project_id: str = 'google.com:datcom-data'
+    gcp_project_id: str = 'datcom-import-automation'
     # ID of the Google Cloud project that stores generated CSVs and MCFs. The
     # project needs to enable Cloud Storage and gives the service account the
     # executor uses sufficient permissions to read and write the bucket below.

import-automation/executor/app/executor/cloud_scheduler.py

@@ -26,7 +26,8 @@
 from google.protobuf import json_format
 from google.api_core.exceptions import AlreadyExists, NotFound
 
-GKE_SERVICE_DOMAIN = os.getenv('GKE_SERVICE_DOMAIN', 'import.datacommons.dev')
+GKE_SERVICE_DOMAIN = os.getenv('GKE_SERVICE_DOMAIN',
+                               'importautomation.datacommons.org')
 GKE_CALLER_SERVICE_ACCOUNT = os.getenv('CLOUD_SCHEDULER_CALLER_SA')
 GKE_OAUTH_AUDIENCE = os.getenv('CLOUD_SCHEDULER_CALLER_OAUTH_AUDIENCE')
 

import-automation/executor/gke/README.md

@@ -26,8 +26,14 @@ Follow
 
 ## (One Time) Setup GKE
 
-1. Update OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in "gke/configure_gke.sh".
-2. Run `./gke/configure_gke.sh`.
+1. Set the PROJECT_ID, OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET environment variables in "gke/configure_gke.sh", e.g.
+```
+export PROJECT_ID=<GCP_PROJECT>
+export PROJECT_ID=<OAUTH_CLIENT_ID>
+export PROJECT_ID=<OAUTH_CLIENT_SECRET>
+```
+
+2. Run `./gke/configure_gke.sh`. The script will error out if the environment variables in (1) are not set.
 
 ## Deployment
 

import-automation/executor/gke/configure_gke.sh

@@ -14,7 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PROJECT_ID=datcom-data
+# Verify that the required environment variables are set.
+if [ -z "$PROJECT_ID" ]
+then
+      echo "\$PROJECT_ID must be set and cannot be empty."
+      exit 1
+fi
+if [ -z "$OAUTH_CLIENT_ID" ]
+then
+      echo "\$OAUTH_CLIENT_ID must be set and cannot be empty."
+      exit 1
+fi
+if [ -z "$OAUTH_CLIENT_SECRET" ]
+then
+      echo "\$OAUTH_CLIENT_SECRET must be set and cannot be empty."
+      exit 1
+fi
+
+PROJECT_ID=datcom-import-automation
 
 gcloud config set project $PROJECT_ID
 
@@ -40,21 +57,16 @@ kubectl create namespace import-automation \
 kubectl create serviceaccount --namespace import-automation import-automation-ksa \
   --dry-run=client -o yaml | kubectl apply -f -
 
-gcloud iam service-accounts add-iam-policy-binding \
-  --project $PROJECT_ID \
-  --role roles/iam.workloadIdentityUser \
-  --member "serviceAccount:$PROJECT_ID.svc.id.goog[import-automation/import-automation-ksa]" \
-  [email protected]
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --role=roles/iam.workloadIdentityUser \
+  --member="serviceAccount:$PROJECT_ID.svc.id.goog[import-automation/import-automation-ksa]"
 
 kubectl annotate serviceaccount \
   --namespace import-automation \
   --overwrite \
   import-automation-ksa \
   iam.gke.io/[email protected]
 
-# Set the oauth env vars before running the script
-# export OAUTH_CLIENT_ID=<fill>
-# export OAUTH_CLIENT_SECRET=<fill>
 kubectl -n import-automation create secret generic import-automation-iap-secret \
   --from-literal=client_id=$OAUTH_CLIENT_ID \
   --from-literal=client_secret=$OAUTH_CLIENT_SECRET

import-automation/executor/requirements.txt

@@ -9,3 +9,4 @@ flask
 gunicorn
 pytz
 absl-py
+libwebp-dev>=1.3.2  # To fix a zero-day vulnerability (CVE-2023-4863): https://snyk.io/blog/find-and-fix-webp-vulnerability-cve-2023-4863/

import-automation/executor/test/cloud_scheduler_test.py

@@ -84,7 +84,7 @@ def test_http_job_request(self):
                 'seconds': 60 * 30
             },
             'http_target': {
-                'uri': 'https://import.datacommons.dev/update',
+                'uri': 'https://importautomation.datacommons.org/update',
                 'http_method': 'POST',
                 'headers': {
                     'Content-Type': 'application/json',

import-automation/executor/test/integration_test.py

@@ -24,10 +24,13 @@
 NUM_LINES_TO_CHECK = 50
 
 CONFIGS = {
-    'github_repo_owner_username': os.environ['GITHUB_AUTH_USERNAME'],
+    # The GitHub params belong to the public Data Commons gmail account.
+    # Auth tokens, user name and other details can be found in the inbox
+    # and in the inbox of teammates.
+    'github_repo_owner_username': os.environ['_GITHUB_REPO_OWNER_USERNAME'],
     'github_repo_name': 'data-demo',
-    'github_auth_username': 'intrepiditee',
-    'github_auth_access_token': os.environ['GITHUB_AUTH_ACCESS_TOKEN']
+    'github_auth_username': os.environ['_GITHUB_AUTH_USERNAME'],
+    'github_auth_access_token': os.environ['_GITHUB_AUTH_ACCESS_TOKEN']
 }