remove underscore from suffix in env.sh.example and adjust pipelines …

…to add it for BigQuery tables
teamdatatonic · Jun 27, 2023 · 25a3990 · 25a3990
1 parent 600e3e9
commit 25a3990
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 5 deletions.
diff --git a/cloudbuild/e2e-test.yaml b/cloudbuild/e2e-test.yaml
@@ -45,7 +45,7 @@ steps:
       - PIPELINE_TEMPLATE=${_PIPELINE_TEMPLATE}
       - VERTEX_PIPELINE_ROOT=${_TEST_VERTEX_PIPELINE_ROOT}
       - PIPELINE_FILES_GCS_PATH=${_PIPELINE_PUBLISH_GCS_PATH}/${COMMIT_SHA}
-      - RESOURCE_SUFFIX=_${COMMIT_SHA}
+      - RESOURCE_SUFFIX=${COMMIT_SHA}
 
 options:
   logging: CLOUD_LOGGING_ONLY

diff --git a/env.sh.example b/env.sh.example
@@ -21,8 +21,8 @@ export VERTEX_LOCATION=europe-west2
 export VERTEX_NETWORK= # optional
 export VERTEX_PROJECT_ID=my-gcp-project
 
-# Suffix (e.g. '_<your name>') to facilitate running concurrent pipelines in the same Google Cloud project. Change if working in a team to avoid overwriting resources during development 
-export RESOURCE_SUFFIX=_default
+# Suffix (e.g. '<your name>') to facilitate running concurrent pipelines in the same Google Cloud project. Change if working in a team to avoid overwriting resources during development 
+export RESOURCE_SUFFIX=default
 
 # Leave as-is
 export VERTEX_SA_EMAIL=vertex-pipelines@${VERTEX_PROJECT_ID}.iam.gserviceaccount.com

diff --git a/pipelines/src/pipelines/tensorflow/training/pipeline.py b/pipelines/src/pipelines/tensorflow/training/pipeline.py
@@ -75,7 +75,7 @@ def tensorflow_pipeline(
     label_column_name = "total_fare"
     time_column = "trip_start_timestamp"
     ingestion_table = "taxi_trips"
-    table_suffix = "_tf_training" + str(resource_suffix)  # suffix to table names
+    table_suffix = f"_tf_training_{resource_suffix}"  # suffix to table names
     ingested_table = "ingested_data" + table_suffix
     preprocessed_table = "preprocessed_data" + table_suffix
     train_table = "train_data" + table_suffix

diff --git a/pipelines/src/pipelines/xgboost/training/pipeline.py b/pipelines/src/pipelines/xgboost/training/pipeline.py
@@ -74,7 +74,7 @@ def xgboost_pipeline(
     label_column_name = "total_fare"
     time_column = "trip_start_timestamp"
     ingestion_table = "taxi_trips"
-    table_suffix = "_xgb_training" + str(resource_suffix)  # suffix to table names
+    table_suffix = f"_xgb_training_{resource_suffix}"  # suffix to table names
     ingested_table = "ingested_data" + table_suffix
     preprocessed_table = "preprocessed_data" + table_suffix
     train_table = "train_data" + table_suffix