From bd25a86cd8c493f84f6f3ac705ce2e833f5bb86d Mon Sep 17 00:00:00 2001 From: Karim Wadie Date: Mon, 27 Feb 2023 10:15:36 +0100 Subject: [PATCH] added prepare_backup_projects_for_terraform.sh and update README --- README.md | 17 ++++++++-- .../prepare_backup_projects_for_terraform.sh | 32 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100755 scripts/prepare_backup_projects_for_terraform.sh diff --git a/README.md b/README.md index 017985d..832aac5 100644 --- a/README.md +++ b/README.md @@ -415,7 +415,9 @@ BigQuery Types to Avro Logical Types mapping: | `TIME` | `timestamp-micro` (annotates Avro `LONG`) | | `DATETIME` | `STRING` (custom named logical type `datetime`) | -##### Configure Additional Backup Projects +##### Configure Backup Projects + +###### Additional Backup Projects Terraform needs to deploy resources to the backup projects where the backup operations will run. For example, log sinks that send notifications to the Tagger once a backup operation has completed. @@ -430,6 +432,18 @@ additional_backup_projects = ["project1", "project2", ..] If you're only using the fallback backup policy and without table-level external policies, you can set this variable to an empty list `[]` +###### Configure Terraform SA permissions on Backup Projects + +In order for Terraform to deploy resources on the backup projects (configured in the previous step), the service account +used by Terraform must have the required permissions on these projects. To do so, run the following command: + +```shell +./scripts/prepare_backup_projects_for_terraform.sh +``` + +The list of projects must include all projects you're planning to store backups in. This includes all projects listed under the `backup_project` +field in the fallback policy, plus the ones included in the `additional_backup_projects` Terraform variable. + #### Terraform Deployment ```shell @@ -446,7 +460,6 @@ terraform apply -var-file=$VARS -auto-approve ``` - #### Setup Access to Sources and Destinations ##### Set Environment Variables diff --git a/scripts/prepare_backup_projects_for_terraform.sh b/scripts/prepare_backup_projects_for_terraform.sh new file mode 100755 index 0000000..555be49 --- /dev/null +++ b/scripts/prepare_backup_projects_for_terraform.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# run this script for all projects that are used to store backups +set -e + +for project in "$@" +do + + echo "Preparing backup project ${project} for Terraform .." + + # Terraform needs to create log sinks to capture GCS export operation completion + gcloud projects add-iam-policy-binding "${project}" \ + --member="serviceAccount:${TF_SA}@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/logging.configWriter" + +done