Improve Minimal Data Platform Blueprint (#1473)

* Add SA to access to Curated resources * Add BQ dataset in the landing project * Provide example to move data from landing to curated using BQ engine * Improve diagram
GoogleCloudPlatform · Jun 28, 2023 · 099ad03 · 099ad03
1 parent 6fcb010
commit 099ad03
Show file tree

Hide file tree

Showing 10 changed files with 196 additions and 74 deletions.
diff --git a/blueprints/data-solutions/data-platform-minimal/01-landing.tf b/blueprints/data-solutions/data-platform-minimal/01-landing.tf
@@ -16,7 +16,7 @@
 
 locals {
   iam_lnd = {
-    "roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
+    "roles/storage.objectCreator" = [module.land-sa-0.iam_email]
     "roles/storage.objectViewer"  = [module.processing-sa-cmp-0.iam_email]
     "roles/storage.objectAdmin"   = [module.processing-sa-0.iam_email]
   }
@@ -36,6 +36,9 @@ module "land-project" {
   iam          = var.project_config.billing_account_id != null ? local.iam_lnd : null
   iam_additive = var.project_config.billing_account_id == null ? local.iam_lnd : null
   services = [
+    "bigquery.googleapis.com",
+    "bigqueryreservation.googleapis.com",
+    "bigquerystorage.googleapis.com",
     "cloudkms.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "iam.googleapis.com",
@@ -52,12 +55,12 @@ module "land-project" {
 
 # Cloud Storage
 
-module "land-sa-cs-0" {
+module "land-sa-0" {
   source       = "../../../modules/iam-service-account"
   project_id   = module.land-project.project_id
   prefix       = var.prefix
-  name         = "lnd-cs-0"
-  display_name = "Data platform GCS landing service account."
+  name         = "lnd-sa-0"
+  display_name = "Data platform landing zone service account."
   iam = {
     "roles/iam.serviceAccountTokenCreator" = [
       local.groups_iam.data-engineers
@@ -75,3 +78,11 @@ module "land-cs-0" {
   encryption_key = var.service_encryption_keys.storage
   force_destroy  = var.data_force_destroy
 }
+
+module "land-bq-0" {
+  source         = "../../../modules/bigquery-dataset"
+  project_id     = module.land-project.project_id
+  id             = "${replace(var.prefix, "-", "_")}_lnd_bq_0"
+  location       = var.location
+  encryption_key = var.service_encryption_keys.bq
+}
diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf
@@ -49,16 +49,14 @@ locals {
   }
   processing_subnet = (
     local.use_shared_vpc
-    ? var.network_config.subnet_self_links.processing_transformation
+    ? var.network_config.subnet_self_link
     : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"]
   )
   processing_vpc = (
     local.use_shared_vpc
     ? var.network_config.network_self_link
     : module.processing-vpc.0.self_link
   )
-
-
 }
 
 module "processing-project" {

diff --git a/blueprints/data-solutions/data-platform-minimal/03-curated.tf b/blueprints/data-solutions/data-platform-minimal/03-curated.tf
@@ -18,33 +18,41 @@ locals {
   cur_iam = {
     "roles/bigquery.dataOwner" = [module.processing-sa-0.iam_email]
     "roles/bigquery.dataViewer" = [
+      module.cur-sa-0.iam_email,
       local.groups_iam.data-analysts,
       local.groups_iam.data-engineers
     ]
     "roles/bigquery.jobUser" = [
-      module.processing-sa-0.iam_email,
+      module.processing-sa-0.iam_email, # Remove once bug is fixed. https://github.com/apache/airflow/issues/32106
+      module.cur-sa-0.iam_email,
       local.groups_iam.data-analysts,
       local.groups_iam.data-engineers
     ]
     "roles/datacatalog.tagTemplateViewer" = [
-      local.groups_iam.data-analysts, local.groups_iam.data-engineers
+      module.cur-sa-0.iam_email,
+      local.groups_iam.data-analysts,
+      local.groups_iam.data-engineers
     ]
     "roles/datacatalog.viewer" = [
-      local.groups_iam.data-analysts, local.groups_iam.data-engineers
+      module.cur-sa-0.iam_email,
+      local.groups_iam.data-analysts,
+      local.groups_iam.data-engineers
     ]
     "roles/storage.objectViewer" = [
-      local.groups_iam.data-analysts, local.groups_iam.data-engineers
+      module.cur-sa-0.iam_email,
+      local.groups_iam.data-analysts,
+      local.groups_iam.data-engineers
     ]
     "roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
   }
   cur_services = [
-    "iam.googleapis.com",
     "bigquery.googleapis.com",
     "bigqueryreservation.googleapis.com",
     "bigquerystorage.googleapis.com",
     "cloudkms.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "compute.googleapis.com",
+    "iam.googleapis.com",
     "servicenetworking.googleapis.com",
     "serviceusage.googleapis.com",
     "stackdriver.googleapis.com",
@@ -75,6 +83,19 @@ module "cur-project" {
   }
 }
 
+module "cur-sa-0" {
+  source       = "../../../modules/iam-service-account"
+  project_id   = module.cur-project.project_id
+  prefix       = var.prefix
+  name         = "cur-sa-0"
+  display_name = "Data platform curated zone service account."
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = [
+      local.groups_iam.data-engineers
+    ]
+  }
+}
+
 # Bigquery
 
 module "cur-bq-0" {

diff --git a/blueprints/data-solutions/data-platform-minimal/IAM.md b/blueprints/data-solutions/data-platform-minimal/IAM.md
@@ -9,7 +9,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
 |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/dlp.estimatesAdmin](https://cloud.google.com/iam/docs/understanding-roles#dlp.estimatesAdmin) <br>[roles/dlp.reader](https://cloud.google.com/iam/docs/understanding-roles#dlp.reader) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
 |<b>gcp-data-security</b><br><small><i>group</i></small>|[roles/datacatalog.admin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.admin) <br>[roles/dlp.admin](https://cloud.google.com/iam/docs/understanding-roles#dlp.admin) |
-|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
+|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
 
 ## Project <i>cur</i>
 
@@ -18,15 +18,16 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
 |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
 |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
-|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|<b>cur-sa-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
 
 ## Project <i>lnd</i>
 
 | members | roles |
 |---|---|
-|<b>lnd-cs-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
+|<b>lnd-sa-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
+|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
 |<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
 
 ## Project <i>prc</i>
 
@@ -35,5 +36,5 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
 |<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
 |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
-|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/dataproc.editor](https://cloud.google.com/iam/docs/understanding-roles#dataproc.editor) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|<b>prc-dp-0</b><br><small><i>serviceAccount</i></small>|[roles/dataproc.worker](https://cloud.google.com/iam/docs/understanding-roles#dataproc.worker) |
+|<b>prc-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker) <br>[roles/dataproc.worker](https://cloud.google.com/iam/docs/understanding-roles#dataproc.worker) |
+|<b>prc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) <br>[roles/dataproc.editor](https://cloud.google.com/iam/docs/understanding-roles#dataproc.editor) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |