From 1d7ea743159ec3aaa14262e43ba108657a40a2d7 Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Wed, 28 Jul 2021 23:32:49 +0200 Subject: [PATCH] Transfer Bigquery dataset to k8s-infra-public-pii Create Bigquery scheduled job running every 24h to copy dataset riaan_data_store from k8s-infra-ii-sandbox to dataset k8s_infra_artifacts_gcslogs. Signed-off-by: Arnaud Meukam --- .../projects/k8s-infra-public-pii/bigquery.tf | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 infra/gcp/clusters/projects/k8s-infra-public-pii/bigquery.tf diff --git a/infra/gcp/clusters/projects/k8s-infra-public-pii/bigquery.tf b/infra/gcp/clusters/projects/k8s-infra-public-pii/bigquery.tf new file mode 100644 index 00000000000..e2a2b5353c4 --- /dev/null +++ b/infra/gcp/clusters/projects/k8s-infra-public-pii/bigquery.tf @@ -0,0 +1,34 @@ + +// Service account dedicated for BigQuery Data Transfer +resource "google_service_account" "bq_data_transfer_writer" { + account_id = "bq-data-transfer" + description = "Service Acccount BigQuery Data Transfer" +} + +// grant bigquery dataEditor role to the service account so that scheduled query can run +resource "google_project_iam_member" "bq_data_transfer_writer_binding" { + project = google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.bq_data_transfer_writer.email}" +} + +resource "google_bigquery_data_transfer_config" "bq_data_transfer" { + display_name = "BigQuey data transfer to ${google_bigquery_dataset.audit-logs-gcs.dataset_id}" + project = google_project.project.project_id + data_source_id = "k8s-infra-ii-sandbox.riaan_data_store" + schedule = "every 24 hours" #Times are in UTC + destination_dataset_id = google_bigquery_dataset.audit-logs-gcs.dataset_id + service_account_name = google_service_account.bq_data_transfer_writer.email + + params = { + write_disposition = "WRITE_TRUNCATE" #Overwrite existing data + } + + schedule_options { + start_time = "15h00" #in UTC + } + + email_preferences { + enable_failure_email = true + } +}