Merge pull request #1089 from GoogleCloudPlatform/lcaggio/dp-20230111

Update Data Platform

blueprints/data-solutions/data-platform-foundations/03-composer.tf

@@ -14,6 +14,40 @@
 
 # tfdoc:file:description Orchestration Cloud Composer definition.
 
+locals {
+  env_variables = {
+    BQ_LOCATION                 = var.location
+    DATA_CAT_TAGS               = try(jsonencode(module.common-datacatalog.tags), "{}")
+    DF_KMS_KEY                  = try(var.service_encryption_keys.dataflow, "")
+    DRP_PRJ                     = module.drop-project.project_id
+    DRP_BQ                      = module.drop-bq-0.dataset_id
+    DRP_GCS                     = module.drop-cs-0.url
+    DRP_PS                      = module.drop-ps-0.id
+    DWH_LAND_PRJ                = module.dwh-lnd-project.project_id
+    DWH_LAND_BQ_DATASET         = module.dwh-lnd-bq-0.dataset_id
+    DWH_LAND_GCS                = module.dwh-lnd-cs-0.url
+    DWH_CURATED_PRJ             = module.dwh-cur-project.project_id
+    DWH_CURATED_BQ_DATASET      = module.dwh-cur-bq-0.dataset_id
+    DWH_CURATED_GCS             = module.dwh-cur-cs-0.url
+    DWH_CONFIDENTIAL_PRJ        = module.dwh-conf-project.project_id
+    DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
+    DWH_CONFIDENTIAL_GCS        = module.dwh-conf-cs-0.url
+    GCP_REGION                  = var.region
+    LOD_PRJ                     = module.load-project.project_id
+    LOD_GCS_STAGING             = module.load-cs-df-0.url
+    LOD_NET_VPC                 = local.load_vpc
+    LOD_NET_SUBNET              = local.load_subnet
+    LOD_SA_DF                   = module.load-sa-df-0.email
+    ORC_PRJ                     = module.orch-project.project_id
+    ORC_GCS                     = module.orch-cs-0.url
+    TRF_PRJ                     = module.transf-project.project_id
+    TRF_GCS_STAGING             = module.transf-cs-df-0.url
+    TRF_NET_VPC                 = local.transf_vpc
+    TRF_NET_SUBNET              = local.transf_subnet
+    TRF_SA_DF                   = module.transf-sa-df-0.email
+    TRF_SA_BQ                   = module.transf-sa-bq-0.email
+  }
+}
 module "orch-sa-cmp-0" {
   source       = "../../../modules/iam-service-account"
   project_id   = module.orch-project.project_id
@@ -27,21 +61,51 @@ module "orch-sa-cmp-0" {
 }
 
 resource "google_composer_environment" "orch-cmp-0" {
-  provider = google-beta
-  project  = module.orch-project.project_id
-  name     = "${var.prefix}-orc-cmp-0"
-  region   = var.region
+  count   = var.composer_config.disable_deployment == true ? 0 : 1
+  project = module.orch-project.project_id
+  name    = "${var.prefix}-orc-cmp-0"
+  region  = var.region
   config {
-    node_count = var.composer_config.node_count
+    software_config {
+      airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null)
+      pypi_packages            = try(var.composer_config.software_config.pypi_packages, null)
+      env_variables            = merge(try(var.composer_config.software_config.env_variables, null), local.env_variables)
+      image_version            = try(var.composer_config.software_config.image_version, null)
+    }
+    dynamic "workloads_config" {
+      for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {})
+
+      content {
+        scheduler {
+          cpu        = try(var.composer_config.workloads_config.scheduler.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.scheduler.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.scheduler.storage_gb, null)
+          count      = try(var.composer_config.workloads_config.scheduler.count, null)
+        }
+        web_server {
+          cpu        = try(var.composer_config.workloads_config.web_server.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.web_server.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.web_server.storage_gb, null)
+        }
+        worker {
+          cpu        = try(var.composer_config.workloads_config.worker.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.worker.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.worker.storage_gb, null)
+          min_count  = try(var.composer_config.workloads_config.worker.min_count, null)
+          max_count  = try(var.composer_config.workloads_config.worker.max_count, null)
+        }
+      }
+    }
+
+    environment_size = var.composer_config.environment_size
+
     node_config {
-      zone                 = "${var.region}-b"
-      service_account      = module.orch-sa-cmp-0.email
       network              = local.orch_vpc
       subnetwork           = local.orch_subnet
-      tags                 = ["composer-worker", "http-server", "https-server"]
-      enable_ip_masq_agent = true
+      service_account      = module.orch-sa-cmp-0.email
+      enable_ip_masq_agent = "true"
+      tags                 = ["composer-worker"]
       ip_allocation_policy {
-        use_ip_aliases = "true"
         cluster_secondary_range_name = try(
           var.network_config.composer_secondary_ranges.pods, "pods"
         )
@@ -58,80 +122,20 @@ resource "google_composer_environment" "orch-cmp-0" {
       master_ipv4_cidr_block = try(
         var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28"
       )
-      web_server_ipv4_cidr_block = try(
-        var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28"
-      )
     }
-    software_config {
-      image_version = var.composer_config.airflow_version
-      env_variables = merge(
-        var.composer_config.env_variables, {
-          BQ_LOCATION                 = var.location
-          DATA_CAT_TAGS               = try(jsonencode(module.common-datacatalog.tags), "{}")
-          DF_KMS_KEY                  = try(var.service_encryption_keys.dataflow, "")
-          DRP_PRJ                     = module.drop-project.project_id
-          DRP_BQ                      = module.drop-bq-0.dataset_id
-          DRP_GCS                     = module.drop-cs-0.url
-          DRP_PS                      = module.drop-ps-0.id
-          DWH_LAND_PRJ                = module.dwh-lnd-project.project_id
-          DWH_LAND_BQ_DATASET         = module.dwh-lnd-bq-0.dataset_id
-          DWH_LAND_GCS                = module.dwh-lnd-cs-0.url
-          DWH_CURATED_PRJ             = module.dwh-cur-project.project_id
-          DWH_CURATED_BQ_DATASET      = module.dwh-cur-bq-0.dataset_id
-          DWH_CURATED_GCS             = module.dwh-cur-cs-0.url
-          DWH_CONFIDENTIAL_PRJ        = module.dwh-conf-project.project_id
-          DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
-          DWH_CONFIDENTIAL_GCS        = module.dwh-conf-cs-0.url
-          DWH_PLG_PRJ                 = module.dwh-plg-project.project_id
-          DWH_PLG_BQ_DATASET          = module.dwh-plg-bq-0.dataset_id
-          DWH_PLG_GCS                 = module.dwh-plg-cs-0.url
-          GCP_REGION                  = var.region
-          LOD_PRJ                     = module.load-project.project_id
-          LOD_GCS_STAGING             = module.load-cs-df-0.url
-          LOD_NET_VPC                 = local.load_vpc
-          LOD_NET_SUBNET              = local.load_subnet
-          LOD_SA_DF                   = module.load-sa-df-0.email
-          ORC_PRJ                     = module.orch-project.project_id
-          ORC_GCS                     = module.orch-cs-0.url
-          TRF_PRJ                     = module.transf-project.project_id
-          TRF_GCS_STAGING             = module.transf-cs-df-0.url
-          TRF_NET_VPC                 = local.transf_vpc
-          TRF_NET_SUBNET              = local.transf_subnet
-          TRF_SA_DF                   = module.transf-sa-df-0.email
-          TRF_SA_BQ                   = module.transf-sa-bq-0.email
-        }
-      )
-    }
-
     dynamic "encryption_config" {
       for_each = (
-        try(local.service_encryption_keys.composer != null, false)
+        try(var.service_encryption_keys[var.region], null) != null
         ? { 1 = 1 }
         : {}
       )
       content {
-        kms_key_name = try(local.service_encryption_keys.composer, null)
+        kms_key_name = try(var.service_encryption_keys[var.region], null)
       }
     }
-
-    # dynamic "web_server_network_access_control" {
-    #   for_each = toset(
-    #     var.network_config.web_server_network_access_control == null
-    #     ? []
-    #     : [var.network_config.web_server_network_access_control]
-    #   )
-    #   content {
-    #     dynamic "allowed_ip_range" {
-    #       for_each = toset(web_server_network_access_control.key)
-    #       content {
-    #         value = allowed_ip_range.key
-    #       }
-    #     }
-    #   }
-    # }
-
   }
   depends_on = [
     google_project_iam_member.shared_vpc,
+    module.orch-project
   ]
 }

blueprints/data-solutions/data-platform-foundations/03-orchestration.tf

@@ -54,6 +54,9 @@ module "orch-project" {
     "roles/bigquery.jobUser" = [
       module.orch-sa-cmp-0.iam_email,
     ]
+    "roles/composer.ServiceAgentV2Ext" = [
+      "serviceAccount:${module.orch-project.service_accounts.robots.composer}"
+    ]
     "roles/composer.worker" = [
       module.orch-sa-cmp-0.iam_email
     ]
@@ -67,11 +70,6 @@ module "orch-project" {
     "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
   }
   oslogin = false
-  org_policies = {
-    "constraints/compute.requireOsLogin" = {
-      enforce = false
-    }
-  }
   services = concat(var.project_services, [
     "artifactregistry.googleapis.com",
     "bigquery.googleapis.com",

blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf

@@ -30,21 +30,6 @@ locals {
       "roles/storage.objectViewer",
     ]
   }
-  dwh_plg_group_iam = {
-    (local.groups.data-engineers) = [
-      "roles/bigquery.dataEditor",
-      "roles/storage.admin",
-    ],
-    (local.groups.data-analysts) = [
-      "roles/bigquery.dataEditor",
-      "roles/bigquery.jobUser",
-      "roles/bigquery.metadataViewer",
-      "roles/bigquery.user",
-      "roles/datacatalog.viewer",
-      "roles/datacatalog.tagTemplateViewer",
-      "roles/storage.objectAdmin",
-    ]
-  }
   dwh_lnd_iam = {
     "roles/bigquery.dataOwner" = [
       module.load-sa-df-0.iam_email,
@@ -140,21 +125,6 @@ module "dwh-conf-project" {
   }
 }
 
-module "dwh-plg-project" {
-  source          = "../../../modules/project"
-  parent          = var.folder_id
-  billing_account = var.billing_account_id
-  prefix          = var.prefix
-  name            = "dwh-plg${local.project_suffix}"
-  group_iam       = local.dwh_plg_group_iam
-  iam             = {}
-  services        = local.dwh_services
-  service_encryption_key_ids = {
-    bq      = [try(local.service_encryption_keys.bq, null)]
-    storage = [try(local.service_encryption_keys.storage, null)]
-  }
-}
-
 # Bigquery
 
 module "dwh-lnd-bq-0" {
@@ -181,14 +151,6 @@ module "dwh-conf-bq-0" {
   encryption_key = try(local.service_encryption_keys.bq, null)
 }
 
-module "dwh-plg-bq-0" {
-  source         = "../../../modules/bigquery-dataset"
-  project_id     = module.dwh-plg-project.project_id
-  id             = "${replace(var.prefix, "-", "_")}_dwh_plg_bq_0"
-  location       = var.location
-  encryption_key = try(local.service_encryption_keys.bq, null)
-}
-
 # Cloud storage
 
 module "dwh-lnd-cs-0" {
@@ -223,14 +185,3 @@ module "dwh-conf-cs-0" {
   encryption_key = try(local.service_encryption_keys.storage, null)
   force_destroy  = var.data_force_destroy
 }
-
-module "dwh-plg-cs-0" {
-  source         = "../../../modules/gcs"
-  project_id     = module.dwh-plg-project.project_id
-  prefix         = var.prefix
-  name           = "dwh-plg-cs-0"
-  location       = var.location
-  storage_class  = "MULTI_REGIONAL"
-  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
-}

blueprints/data-solutions/data-platform-foundations/IAM.md

@@ -57,14 +57,6 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 |<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
 |<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |
 
-## Project <i>dwh-plg</i>
-
-| members | roles |
-|---|---|
-|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
-
 ## Project <i>lod</i>
 
 | members | roles |
@@ -80,7 +72,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
 | members | roles |
 |---|---|
 |<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
 |<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
 |<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
 |<b>orc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |