Merge branch 'datacommonsorg:master' into terraform

datacommonsorg · Dec 17, 2024 · 3425dd0 · 3425dd0
2 parents 9ad9dbc + 1d2b0df
commit 3425dd0
Show file tree

Hide file tree

Showing 41 changed files with 510 additions and 793 deletions.
diff --git a/build/ci/cloudbuild.py.yaml b/build/ci/cloudbuild.py.yaml
@@ -14,9 +14,20 @@
 
 # Run Python tests, lint, etc.
 steps:
+  # Download the files needed for nl embeddings tests.
+  - id: download_nl_files
+    name: python:3.11.3
+    entrypoint: /bin/sh
+    args:
+      - -c
+      - |
+        cd tools/nl/download_nl_files
+        ./run.sh
+
   - id: flask_test
     name: python:3.11.3
     entrypoint: /bin/sh
+    waitFor: ["download_nl_files"]
     args:
       - -c
       - |

diff --git a/run_website_sanity.sh b/run_website_sanity.sh
@@ -17,8 +17,22 @@ set -e
 
 # DC Instance domain like: "dev.datacommons.org", "datacommons.org"
 domain=$1
+# Check if the domain is set
+if [[ -z "$domain" ]]; then
+  echo "Error: Domain is not set."
+  echo "Usage: $0 <domain> [NO_PIP]"
+  echo "Example: $0 autopush.datacommons.org"
+  exit 1
+fi
+
+# Prepend https:// if no protocol is specified
+if [[ ! "$domain" =~ ^http:// && ! "$domain" =~ ^https:// ]]; then
+  domain="https://$domain"
+fi
+
 echo "Domain: $domain"
 
+# Set to true if you don't want to install the dependencies
 NO_PIP=$2
 
 export FLASK_ENV=webdriver
@@ -32,6 +46,6 @@ if [[ $NO_PIP != "true" ]]; then
 fi
 
 date_str=$(TZ="America/Los_Angeles" date +"%Y_%m_%d_%H_%M_%S")
-python3 server/webdriver/tests/standalone/sanity.py --mode=home --url="https://$domain"
+python3 server/webdriver/tests/standalone/sanity.py --mode=home --url="$domain"
 gsutil cp ./output/*.csv gs://datcom-website-sanity/$domain/$date_str/
 rm ./output/*.csv
diff --git a/server/integration_tests/test_data/demo_fallback/query_5/chart_config.json b/server/integration_tests/test_data/demo_fallback/query_5/chart_config.json
@@ -627,18 +627,40 @@
               {
                 "tiles": [
                   {
+                    "barTileSpec": {
+                      "maxPlaces": 15,
+                      "maxVariables": 15,
+                      "sort": "DESCENDING"
+                    },
+                    "comparisonPlaces": [
+                      "country/USA"
+                    ],
                     "statVarKey": [
-                      "Amount_EconomicActivity_GrossDomesticProduction_RealValue",
-                      "Amount_EconomicActivity_GrossValueAdded_RealValue"
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSConstruction_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSAccommodationFoodServices_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSAdministrativeSupportWasteManagementRemediationServices_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSAgricultureForestryFishingHunting_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSArtsEntertainmentRecreation_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSEducationalServices_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSFinanceInsurance_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSHealthCareSocialAssistance_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSInformation_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSManagementOfCompaniesEnterprises_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSMiningQuarryingOilGasExtraction_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSOtherServices_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSProfessionalScientificTechnicalServices_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSRealEstateRentalLeasing_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSRetailTrade_RealValue_multiple_place_bar_block",
+                      "Amount_EconomicActivity_GrossDomesticProduction_NAICSTransportationWarehousing_RealValue_multiple_place_bar_block"
                     ],
-                    "title": "GDP (Real Value) and Amount of Economic Activity (Real Value): Gross Value Added in United States",
-                    "type": "LINE"
+                    "title": "GDP of the Construction Industry and More in United States (${date})",
+                    "type": "BAR"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Related: GDP (Real Value) and Amount of Economic Activity (Real Value): Gross Value Added"
+            "title": "Related: GDP of the Construction Industry and More"
           }
         ],
         "statVarSpec": {
@@ -698,6 +720,14 @@
             "name": "GDP of the Real Estate and Rental and Leasing Industry",
             "statVar": "Amount_EconomicActivity_GrossDomesticProduction_NAICSRealEstateRentalLeasing_RealValue"
           },
+          "Amount_EconomicActivity_GrossDomesticProduction_NAICSRetailTrade_RealValue_multiple_place_bar_block": {
+            "name": "Amount of Economic Activity (Real Value): Gross Domestic Production, Retail Trade (NAICS/44-45)",
+            "statVar": "Amount_EconomicActivity_GrossDomesticProduction_NAICSRetailTrade_RealValue"
+          },
+          "Amount_EconomicActivity_GrossDomesticProduction_NAICSTransportationWarehousing_RealValue_multiple_place_bar_block": {
+            "name": "Amount of Economic Activity (Real Value): Gross Domestic Production, Transportation And Warehousing (NAICS/48-49)",
+            "statVar": "Amount_EconomicActivity_GrossDomesticProduction_NAICSTransportationWarehousing_RealValue"
+          },
           "Amount_EconomicActivity_GrossDomesticProduction_NAICSUtilities_RealValue_multiple_place_bar_block": {
             "name": "GDP of the Utilities Industry",
             "statVar": "Amount_EconomicActivity_GrossDomesticProduction_NAICSUtilities_RealValue"
@@ -718,10 +748,6 @@
             "name": "GDP (Real Value)",
             "statVar": "Amount_EconomicActivity_GrossDomesticProduction_RealValue"
           },
-          "Amount_EconomicActivity_GrossValueAdded_RealValue": {
-            "name": "Amount of Economic Activity (Real Value): Gross Value Added",
-            "statVar": "Amount_EconomicActivity_GrossValueAdded_RealValue"
-          },
           "Area_Farm": {
             "name": "Total Farm Area",
             "statVar": "Area_Farm"
@@ -1210,17 +1236,6 @@
           "Amount_EconomicActivity_GrossDomesticProduction_NAICSWholesaleTrade_RealValue"
         ]
       },
-      "Amount_EconomicActivity_GrossValueAdded_RealValue": {
-        "isic": [
-          "Amount_EconomicActivity_GrossValueAdded_ISICAgricultureHuntingForestryFishing_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICConstruction_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICManufacturing_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICMiningManufacturingUtilities_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICOtherActivities_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICTransportStorageCommunications_RealValue",
-          "Amount_EconomicActivity_GrossValueAdded_ISICWholesaleRetailTradeRestaurantsHotels_RealValue"
-        ]
-      },
       "dc/1wf1h5esex2d": {
         "establishmentOwnership": [
           "dc/n1mgflew2xqm8",