fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup!…

… Copy PostgresSQL to GCS as CSV
KlubJagiellonski · Apr 18, 2024 · 0c29b92 · 0c29b92
1 parent 4d0a94c
commit 0c29b92
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/.github/workflows/bi_transfer_pola_backend_to_bq.yml b/.github/workflows/bi_transfer_pola_backend_to_bq.yml
@@ -81,4 +81,4 @@ jobs:
           --target-location "${GCP_REGION}"
           --target-dataset "${GCP_BIGQUERY_DATASET}"
       - name: "List BigQuery tables"
-        run: bq ls pola-bi-looker:pola-backend
+        run: bq ls "${GCP_BIGQUERY_DATASET}"
diff --git a/pola-bi/postgres_to_gcs/gcs_to_bigquery.py b/pola-bi/postgres_to_gcs/gcs_to_bigquery.py
@@ -59,8 +59,8 @@ def main():
         subprocess.check_output(['gcloud', 'storage', 'ls', '--recursive', f'{source}/**/*.csv']).decode().splitlines()
     )
 
-    for csv_file in csv_files:
-        log.info("Loading file: %s", csv_file)
+    for no, csv_file in enumerate(csv_files, start=1):
+        log.info("Loading file %s (%d/%d)", csv_file, no, len(csv_files))
         table_name = csv_file.split('/')[-1].split('.')[0]
 
         with NamedTemporaryFile() as f: