Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix BigQuery Transfer config #2415

Merged
merged 2 commits into from
Jul 29, 2021
Merged

Fix BigQuery Transfer config #2415

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d984b6
Add missing required field for service account
ameukam Jul 29, 2021
045b60d
Fix resource for bigquery transfer
ameukam Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions infra/gcp/clusters/projects/k8s-infra-public-pii/bigquery.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
resource "google_service_account" "bq_data_transfer_writer" {
account_id = "bq-data-transfer"
description = "Service Acccount BigQuery Data Transfer"
project = google_project.project.project_id
}

// grant bigquery dataEditor role to the service account so that scheduled query can run
Expand All @@ -15,17 +16,21 @@ resource "google_project_iam_member" "bq_data_transfer_writer_binding" {
resource "google_bigquery_data_transfer_config" "bq_data_transfer" {
display_name = "BigQuey data transfer to ${google_bigquery_dataset.audit-logs-gcs.dataset_id}"
project = google_project.project.project_id
data_source_id = "k8s-infra-ii-sandbox.riaan_data_store"
data_source_id = "cross_region_copy"
schedule = "every 24 hours" #Times are in UTC
destination_dataset_id = google_bigquery_dataset.audit-logs-gcs.dataset_id
service_account_name = google_service_account.bq_data_transfer_writer.email
disabled = false

params = {
write_disposition = "WRITE_TRUNCATE" #Overwrite existing data
overwrite_destination_table = "true"
source_dataset_id = "riaan_data_store"
source_project_id = "k8s-infra-ii-sandbox"
}

schedule_options {
start_time = "15h00" #in UTC
disable_auto_scheduling = false
start_time = "2021-07-29T15:00:00Z"
}

email_preferences {
Expand Down