Import automation: Setup script updates

import-automation/executor/Dockerfile

@@ -16,7 +16,10 @@
 
 FROM python:3.11.4
 
-RUN apt upgrade
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /workspace
 
 ADD requirements.txt /workspace/requirements.txt

import-automation/executor/app/configs.py

@@ -34,7 +34,7 @@ class ExecutorConfig:
 
     # ID of the Google Cloud project that hosts the executor. The project
     # needs to enable App Engine and Cloud Scheduler.
-    gcp_project_id: str = 'google.com:datcom-data'
+    gcp_project_id: str = 'google.com:datcom-import-automation'
     # ID of the Google Cloud project that stores generated CSVs and MCFs. The
     # project needs to enable Cloud Storage and gives the service account the
     # executor uses sufficient permissions to read and write the bucket below.

import-automation/executor/app/executor/cloud_scheduler.py

@@ -26,7 +26,8 @@
 from google.protobuf import json_format
 from google.api_core.exceptions import AlreadyExists, NotFound
 
-GKE_SERVICE_DOMAIN = os.getenv('GKE_SERVICE_DOMAIN', 'import.datacommons.dev')
+GKE_SERVICE_DOMAIN = os.getenv('GKE_SERVICE_DOMAIN',
+                               'importautomation.datacommons.org')
 GKE_CALLER_SERVICE_ACCOUNT = os.getenv('CLOUD_SCHEDULER_CALLER_SA')
 GKE_OAUTH_AUDIENCE = os.getenv('CLOUD_SCHEDULER_CALLER_OAUTH_AUDIENCE')
 

import-automation/executor/gke/README.md

@@ -26,8 +26,11 @@ Follow
 
 ## (One Time) Setup GKE
 
-1. Update OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in "gke/configure_gke.sh".
-2. Run `./gke/configure_gke.sh`.
+1. Update the PROJECT_ID in "gke/configure_gke.sh", if needed.
+
+2. Update OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET in "gke/configure_gke.sh".
+
+3. Run `./gke/configure_gke.sh`.
 
 ## Deployment
 

import-automation/executor/gke/configure_gke.sh

@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PROJECT_ID=datcom-data
+# Edit the PROJECT_ID below.
+PROJECT_ID=datcom-import-automation
 
 gcloud config set project $PROJECT_ID
 
@@ -40,11 +41,9 @@ kubectl create namespace import-automation \
 kubectl create serviceaccount --namespace import-automation import-automation-ksa \
   --dry-run=client -o yaml | kubectl apply -f -
 
-gcloud iam service-accounts add-iam-policy-binding \
-  --project $PROJECT_ID \
-  --role roles/iam.workloadIdentityUser \
-  --member "serviceAccount:$PROJECT_ID.svc.id.goog[import-automation/import-automation-ksa]" \
-  [email protected]
+gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --role=roles/iam.workloadIdentityUser \
+  --member="serviceAccount:$PROJECT_ID.svc.id.goog[import-automation/import-automation-ksa]"
 
 kubectl annotate serviceaccount \
   --namespace import-automation \
@@ -53,8 +52,8 @@ kubectl annotate serviceaccount \
   iam.gke.io/[email protected]
 
 # Set the oauth env vars before running the script
-# export OAUTH_CLIENT_ID=<fill>
-# export OAUTH_CLIENT_SECRET=<fill>
+export OAUTH_CLIENT_ID=
+export OAUTH_CLIENT_SECRET=
 kubectl -n import-automation create secret generic import-automation-iap-secret \
   --from-literal=client_id=$OAUTH_CLIENT_ID \
   --from-literal=client_secret=$OAUTH_CLIENT_SECRET

import-automation/executor/requirements.txt

@@ -9,3 +9,4 @@ flask
 gunicorn
 pytz
 absl-py
+libwebp-dev>=1.3.2  # To fix a zero-day vulnerability (CVE-2023-4863): https://snyk.io/blog/find-and-fix-webp-vulnerability-cve-2023-4863/

import-automation/executor/test/cloud_scheduler_test.py

@@ -84,7 +84,7 @@ def test_http_job_request(self):
                 'seconds': 60 * 30
             },
             'http_target': {
-                'uri': 'https://import.datacommons.dev/update',
+                'uri': 'https://importautomation.datacommons.org/update',
                 'http_method': 'POST',
                 'headers': {
                     'Content-Type': 'application/json',

import-automation/executor/test/integration_test.py

@@ -24,10 +24,11 @@
 NUM_LINES_TO_CHECK = 50
 
 CONFIGS = {
-    'github_repo_owner_username': os.environ['GITHUB_AUTH_USERNAME'],
+    # TODO: read the params from the environment.
+    'github_repo_owner_username': os.environ['_GITHUB_AUTH_USERNAME'],
     'github_repo_name': 'data-demo',
-    'github_auth_username': 'intrepiditee',
-    'github_auth_access_token': os.environ['GITHUB_AUTH_ACCESS_TOKEN']
+    'github_auth_username': '[email protected]',
+    'github_auth_access_token': os.environ['_GITHUB_AUTH_ACCESS_TOKEN']
 }