2024-11-06 Custom DC stable release (#4710)

Highlights:
- Pins the version of `transformers` in nl_requirements.txt
- Adds support for a schema update mode in the data management
container, documented
[here](https://docs.datacommons.org/custom_dc/troubleshooting.html#schema-check-failed).
Schema check errors should also now have a direct link to the
troubleshooting page.
- Updates services container to exit as soon as any of the mixer, NL, or
website servers fails to start up

.github/workflows/all-commits-in-master.yml

@@ -0,0 +1,34 @@
+name: all-commits-in-master
+
+on:
+  pull_request:
+    branches: [ "customdc_stable" ]
+
+jobs:
+  check_commits:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Fetch all history for accurate comparison
+          fetch-depth: 0
+          # Check out the PR branch
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Check if commits exist in master
+        run: |
+          git remote add dc https://github.com/datacommonsorg/website.git
+          git fetch dc
+          MASTER_BRANCH="dc/master"
+
+          # Get the list of commits in the source branch that are not in the master branch
+          MISSING_COMMITS=$(git log --pretty="%H - %s" $MASTER_BRANCH..HEAD --)
+
+          if [[ -n "$MISSING_COMMITS" ]]; then
+            echo "ERROR: The following commits are not present in $MASTER_BRANCH:"
+            echo "$MISSING_COMMITS"
+            exit 1
+          fi
+
+          echo "All commits are present in $MASTER_BRANCH"

build/cdc_data/run.sh

@@ -32,6 +32,16 @@ if [[ $OUTPUT_DIR == "" ]]; then
     exit 1
 fi
 
+if [[ $DATA_RUN_MODE != "" ]]; then
+    if [[ $DATA_RUN_MODE != "schemaupdate" ]]; then
+      echo "DATA_RUN_MODE must be either empty or 'schemaupdate'"
+      exit 1
+    fi
+    echo "DATA_RUN_MODE=$DATA_RUN_MODE"
+else
+  DATA_RUN_MODE="customdc"
+fi
+
 echo "INPUT_DIR=$INPUT_DIR"
 echo "OUTPUT_DIR=$OUTPUT_DIR"
 
@@ -51,7 +61,7 @@ ADDITIONAL_CATALOG_PATH=$DC_NL_EMBEDDINGS_DIR/custom_catalog.yaml
 CUSTOM_EMBEDDINGS_INDEX=user_all_minilm_mem
 
 # Set IS_CUSTOM_DC var to true.
-# This is used by the embeddings builder to set up a custom dc env. 
+# This is used by the embeddings builder to set up a custom dc env.
 export IS_CUSTOM_DC=true
 
 if [[ $USE_SQLITE == "true" ]]; then
@@ -67,15 +77,20 @@ cd $WORKSPACE_DIR/import/simple
 # Run importer.
 python3 -m stats.main \
     --input_dir=$INPUT_DIR \
-    --output_dir=$DC_OUTPUT_DIR
+    --output_dir=$DC_OUTPUT_DIR \
+    --mode=$DATA_RUN_MODE
 
 # cd back to workspace dir to run the embeddings builder.
 cd $WORKSPACE_DIR
 
-# Run embeddings builder.
-python3 -m tools.nl.embeddings.build_embeddings \
-    --embeddings_name=$CUSTOM_EMBEDDINGS_INDEX \
+if [[ $DATA_RUN_MODE == "schemaupdate" ]]; then
+    echo "Skipping embeddings builder because run mode is 'schemaupdate'."
+    echo "Schema update complete."
+else
+    # Run embeddings builder.
+    python3 -m tools.nl.embeddings.build_embeddings \
+        --embeddings_name=$CUSTOM_EMBEDDINGS_INDEX \
     --output_dir=$DC_NL_EMBEDDINGS_DIR \
     --additional_catalog_path=$ADDITIONAL_CATALOG_PATH
-
-echo "Data loading completed."
+  echo "Data loading complete."
+fi

build/cdc_services/run.sh

@@ -88,7 +88,7 @@ else
 fi
 
 # Wait for any process to exit
-wait
+wait -n
 
 # Exit with status of process that exited first
-exit $?
+exit $?

build/ci/cloudbuild.webdriver.yaml

@@ -30,13 +30,26 @@ steps:
         # ./run_test.sh -b will build client packages.
         # These js files generated will be necessery for the flask_webdriver_test task.
         ./run_test.sh -b
+  
+  # Download the files needed for nl server to run. Do the download here because
+  # webdriver runs on mulitple processes & we only want to do the download once.
+  - id: download_nl_files
+    name: python:3.11.3
+    entrypoint: /bin/sh
+    waitFor:
+      - package_js
+    args:
+      - -c
+      - |
+        cd tools/nl/download_nl_files
+        ./run.sh
 
   # Run the webdriver tests
   - id: flask_webdriver_test
     name: gcr.io/datcom-ci/webdriver-chrome:2024-06-05
     entrypoint: /bin/sh
     waitFor:
-      - package_js
+      - download_nl_files
     args:
       - -c
       - |

build/nl_server/Dockerfile

@@ -34,10 +34,10 @@ COPY shared/. /workspace/shared/
 
 # Download nl files from gcs
 COPY deploy/nl/catalog.yaml .
-COPY build/nl_server/requirements.txt /workspace/build/nl_server/requirements.txt
-COPY build/nl_server/download_nl_files.py .
-RUN pip3 install -r /workspace/build/nl_server/requirements.txt
-RUN python3 download_nl_files.py
+COPY tools/nl/download_nl_files/requirements.txt /workspace/download_nl_files/requirements.txt
+COPY tools/nl/download_nl_files/download_nl_files.py .
+RUN pip3 install -r /workspace/download_nl_files/requirements.txt
+RUN python3 download_nl_files.py --is_docker_mode=True
 
 # Run server
 WORKDIR /workspace

deploy/terraform-custom-datacommons/README.md

@@ -50,14 +50,14 @@ dc_api_key  = "your-api-key"
 
 - **project_id**: The Google Cloud project ID where the resources will be created.
 - **namespace**: A unique namespace to differentiate multiple instances of custom Data Commons within the same project.
-- **dc_api_key**: Data Commons API key. [Request an API key](https://docs.google.com/forms/d/e/1FAIpQLSeVCR95YOZ56ABsPwdH1tPAjjIeVDtisLF-8oDYlOxYmNZ7LQ/viewform?resourcekey=0-yJ9nT9ST-TfoKNtmGIws-g)
+- **dc_api_key**: Data Commons API key. [Request an API key](https://apikeys.datacommons.org)
 
 #### Optional Configuration Variables
 
 - **region**: The [GCP region](https://cloud.google.com/about/locations) where resources will be deployed.
 - **enable_redis**: Set to true to enable redis caching (default: false)
-- **dc_web_service_image**: Docker image to use for the services container. Default: `gcr.io/datcom-ci/datacommons-services:stable`
-- **dc_data_job_image**: Docker image to use for the data loading job. Default: `gcr.io/datcom-ci/datacommons-data:stable`
+- **dc_web_service_image**: Docker image to use for the services container. Default: `gcr.io/datcom-ci/datacommons-services:stable`. Set to `gcr.io/datcom-ci/datacommons-services:latest` to use the latest web service image.
+- **dc_data_job_image**: Docker image to use for the data loading job. Default: `gcr.io/datcom-ci/datacommons-data:stable`. Set to `gcr.io/datcom-ci/datacommons-data:latest` to use the latest data job image.
 - **make_dc_web_service_public**: By default, the Data Commons web service is publicly accessible. Set this to `false` if your GCP account has restrictions on public access. [Reference](https://cloud.google.com/run/docs/authenticating/public).
 - **google_analytics_tag_id**: Set to your [Google Analytics Tag ID](https://support.google.com/analytics/answer/9539598) to enable Google Analytics tracking.
 

deploy/terraform-custom-datacommons/modules/main.tf

@@ -169,6 +169,7 @@ resource "google_secret_manager_secret_version" "dc_api_key_version" {
 resource "google_cloud_run_v2_service" "dc_web_service" {
   name     = "${var.namespace}-datacommons-web-service"
   location = var.region
+  deletion_protection = false
 
   template {
     containers {
@@ -320,6 +321,8 @@ resource "google_cloud_run_service_iam_member" "dc_web_service_invoker" {
 resource "google_cloud_run_v2_job" "dc_data_job" {
   name     = "${var.namespace}-datacommons-data-job"
   location = var.region
+  deletion_protection = false
+
   template { 
     template {
       containers {
@@ -375,4 +378,4 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
     google_secret_manager_secret_version.dc_api_key_version,
     google_secret_manager_secret_version.maps_api_key_version
   ]
-}
+}

nl_requirements.txt

@@ -12,4 +12,5 @@ torchvision==0.17.2
 # TODO: this is pinned because latest huggingface_hub is not compatible with
 # sentence-transformers v2.2.2. Look into upgrading sentence-transformers to
 # v2.3.0 or newer
-huggingface_hub==0.25.2
+huggingface_hub==0.25.2
+transformers==4.45.2

server/integration_tests/test_data/demo_fallback/query_4/chart_config.json

@@ -827,6 +827,35 @@
             "denom": "Count_Person",
             "title": "People With Only Private Health Insurance in Counties of California"
           },
+          {
+            "columns": [
+              {
+                "tiles": [
+                  {
+                    "statVarKey": [
+                      "Count_Death"
+                    ],
+                    "title": "Number of Deaths in California",
+                    "type": "LINE"
+                  }
+                ]
+              },
+              {
+                "tiles": [
+                  {
+                    "description": "Number of Deaths in California",
+                    "statVarKey": [
+                      "Count_Death"
+                    ],
+                    "title": "Number of Deaths in California",
+                    "type": "HIGHLIGHT"
+                  }
+                ]
+              }
+            ],
+            "denom": "Count_Person",
+            "title": "Number of Deaths"
+          },
           {
             "columns": [
               {