FAST + Minimal DP

GoogleCloudPlatform · Jan 18, 2024 · e157830 · e157830
1 parent 4a12a33
commit e157830
Show file tree

Hide file tree

Showing 14 changed files with 140 additions and 82 deletions.
diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
@@ -45,7 +45,7 @@ module "drop-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf
@@ -39,7 +39,7 @@ module "load-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -61,7 +61,7 @@ module "orch-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects
@@ -162,8 +162,8 @@ module "orch-cs-df-template" {
   project_id     = module.orch-project.project_id
   prefix         = var.prefix
   name           = "orc-cs-df-template"
-  location       = var.region
-  storage_class  = "REGIONAL"
+  location       = var.location
+  storage_class  = "MULTI_REGIONAL"
   encryption_key = try(local.service_encryption_keys.storage, null)
   force_destroy  = !var.deletion_protection
 }
@@ -173,8 +173,8 @@ module "orch-cs-build-staging" {
   project_id     = module.orch-project.project_id
   prefix         = var.prefix
   name           = "orc-cs-build-staging"
-  location       = var.region
-  storage_class  = "REGIONAL"
+  location       = var.location
+  storage_class  = "MULTI_REGIONAL"
   encryption_key = try(local.service_encryption_keys.storage, null)
   force_destroy  = !var.deletion_protection
 }

diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
@@ -40,7 +40,7 @@ module "transf-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -68,7 +68,7 @@ module "dwh-lnd-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects
@@ -88,7 +88,7 @@ module "dwh-cur-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects
@@ -108,7 +108,7 @@ module "dwh-conf-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf
@@ -50,7 +50,7 @@ module "common-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf
@@ -18,7 +18,7 @@ module "exp-project" {
   source          = "../../../modules/project"
   parent          = var.project_config.parent
   billing_account = var.project_config.billing_account_id
-  project_create  = var.project_config.billing_account_id != null
+  project_create  = var.project_config.project_create
   prefix          = local.use_projects ? null : var.prefix
   name = (
     local.use_projects

diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md
@@ -281,17 +281,17 @@ terraform destroy
 |---|---|:---:|:---:|:---:|
 | [organization_domain](variables.tf#L166) | Organization domain. | <code>string</code> | ✓ |  |
 | [prefix](variables.tf#L171) | Prefix used for resource names. | <code>string</code> | ✓ |  |
-| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; optional&#40;string, null&#41;&#10;  parent             &#61; string&#10;  project_ids &#61; optional&#40;object&#40;&#123;&#10;    drop     &#61; string&#10;    load     &#61; string&#10;    orc      &#61; string&#10;    trf      &#61; string&#10;    dwh-lnd  &#61; string&#10;    dwh-cur  &#61; string&#10;    dwh-conf &#61; string&#10;    common   &#61; string&#10;    exp      &#61; string&#10;    &#125;&#41;, &#123;&#10;    drop     &#61; &#34;drp&#34;&#10;    load     &#61; &#34;lod&#34;&#10;    orc      &#61; &#34;orc&#34;&#10;    trf      &#61; &#34;trf&#34;&#10;    dwh-lnd  &#61; &#34;dwh-lnd&#34;&#10;    dwh-cur  &#61; &#34;dwh-cur&#34;&#10;    dwh-conf &#61; &#34;dwh-conf&#34;&#10;    common   &#61; &#34;cmn&#34;&#10;    exp      &#61; &#34;exp&#34;&#10;    &#125;&#10;  &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
+| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10;  billing_account_id &#61; optional&#40;string, null&#41;&#10;  project_create     &#61; optional&#40;bool, true&#41;&#10;  parent             &#61; string&#10;  project_ids &#61; optional&#40;object&#40;&#123;&#10;    drop     &#61; string&#10;    load     &#61; string&#10;    orc      &#61; string&#10;    trf      &#61; string&#10;    dwh-lnd  &#61; string&#10;    dwh-cur  &#61; string&#10;    dwh-conf &#61; string&#10;    common   &#61; string&#10;    exp      &#61; string&#10;    &#125;&#41;, &#123;&#10;    drop     &#61; &#34;drp&#34;&#10;    load     &#61; &#34;lod&#34;&#10;    orc      &#61; &#34;orc&#34;&#10;    trf      &#61; &#34;trf&#34;&#10;    dwh-lnd  &#61; &#34;dwh-lnd&#34;&#10;    dwh-cur  &#61; &#34;dwh-cur&#34;&#10;    dwh-conf &#61; &#34;dwh-conf&#34;&#10;    common   &#61; &#34;cmn&#34;&#10;    exp      &#61; &#34;exp&#34;&#10;    &#125;&#10;  &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object&#40;&#123;&#10;  disable_deployment &#61; optional&#40;bool&#41;&#10;  environment_size   &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10;  software_config &#61; optional&#40;&#10;    object&#40;&#123;&#10;      airflow_config_overrides       &#61; optional&#40;any&#41;&#10;      pypi_packages                  &#61; optional&#40;any&#41;&#10;      env_variables                  &#61; optional&#40;map&#40;string&#41;&#41;&#10;      image_version                  &#61; string&#10;      cloud_data_lineage_integration &#61; optional&#40;bool, true&#41;&#10;    &#125;&#41;,&#10;    &#123; image_version &#61; &#34;composer-2-airflow-2&#34; &#125;&#10;  &#41;&#10;  workloads_config &#61; optional&#40;&#10;    object&#40;&#123;&#10;      scheduler &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;          count      &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;          count      &#61; 1&#10;        &#125;&#10;      &#41;&#10;      web_server &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;        &#125;&#10;      &#41;&#10;      worker &#61; optional&#40;&#10;        object&#40;&#123;&#10;          cpu        &#61; number&#10;          memory_gb  &#61; number&#10;          storage_gb &#61; number&#10;          min_count  &#61; number&#10;          max_count  &#61; number&#10;        &#125;&#41;,&#10;        &#123;&#10;          cpu        &#61; 0.5&#10;          memory_gb  &#61; 1.875&#10;          storage_gb &#61; 1&#10;          min_count  &#61; 1&#10;          max_count  &#61; 3&#10;        &#125;&#10;      &#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10;  software_config &#61; &#123;&#10;    image_version &#61; &#34;composer-2-airflow-2&#34;&#10;  &#125;&#10;  workloads_config &#61; &#123;&#10;    scheduler &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;      count      &#61; 1&#10;    &#125;&#10;    web_server &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;    &#125;&#10;    worker &#61; &#123;&#10;      cpu        &#61; 0.5&#10;      memory_gb  &#61; 1.875&#10;      storage_gb &#61; 1&#10;      min_count  &#61; 1&#10;      max_count  &#61; 3&#10;    &#125;&#10;  &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
 | [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map&#40;object&#40;&#123;&#10;  description &#61; optional&#40;string&#41;&#10;  iam         &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code title="&#123;&#10;  &#34;3_Confidential&#34; &#61; &#123;&#125;&#10;  &#34;2_Private&#34;      &#61; &#123;&#125;&#10;  &#34;1_Sensitive&#34;    &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
 | [deletion_protection](variables.tf#L120) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | <code>bool</code> |  | <code>false</code> |
 | [groups](variables.tf#L127) | User groups. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  data-analysts  &#61; &#34;gcp-data-analysts&#34;&#10;  data-engineers &#61; &#34;gcp-data-engineers&#34;&#10;  data-security  &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
 | [location](variables.tf#L137) | Location used for multi-regional resources. | <code>string</code> |  | <code>&#34;eu&#34;</code> |
 | [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10;  host_project      &#61; string&#10;  network_self_link &#61; string&#10;  subnet_self_links &#61; object&#40;&#123;&#10;    load           &#61; string&#10;    transformation &#61; string&#10;    orchestration  &#61; string&#10;  &#125;&#41;&#10;  composer_ip_ranges &#61; object&#40;&#123;&#10;    cloudsql   &#61; string&#10;    gke_master &#61; string&#10;  &#125;&#41;&#10;  composer_secondary_ranges &#61; object&#40;&#123;&#10;    pods     &#61; string&#10;    services &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [project_services](variables.tf#L214) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
-| [project_suffix](variables.tf#L225) | Suffix used only for project ids. | <code>string</code> |  | <code>null</code> |
-| [region](variables.tf#L231) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
-| [service_encryption_keys](variables.tf#L237) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [project_services](variables.tf#L215) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> |  | <code title="&#91;&#10;  &#34;cloudresourcemanager.googleapis.com&#34;,&#10;  &#34;iam.googleapis.com&#34;,&#10;  &#34;serviceusage.googleapis.com&#34;,&#10;  &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
+| [project_suffix](variables.tf#L226) | Suffix used only for project ids. | <code>string</code> |  | <code>null</code> |
+| [region](variables.tf#L232) | Region used for regional resources. | <code>string</code> |  | <code>&#34;europe-west1&#34;</code> |
+| [service_encryption_keys](variables.tf#L238) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10;  bq       &#61; string&#10;  composer &#61; string&#10;  dataflow &#61; string&#10;  storage  &#61; string&#10;  pubsub   &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 
 ## Outputs
 

diff --git a/blueprints/data-solutions/data-platform-foundations/main.tf b/blueprints/data-solutions/data-platform-foundations/main.tf
@@ -74,7 +74,7 @@ locals {
       ]
     ]) : "${binding.role}-${binding.member}" => binding
   }
-  use_projects   = var.project_config.billing_account_id == null
+  use_projects   = !var.project_config.project_create
   use_shared_vpc = var.network_config != null
 }
 

diff --git a/blueprints/data-solutions/data-platform-foundations/terraform.tfvars.sample b/blueprints/data-solutions/data-platform-foundations/terraform.tfvars.sample
diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf
@@ -181,6 +181,7 @@ variable "project_config" {
   description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
   type = object({
     billing_account_id = optional(string, null)
+    project_create     = optional(bool, true)
     parent             = string
     project_ids = optional(object({
       drop     = string