Minimal Data Platform - Shared VPC (#1475)

* Fix * Fix dataproc vpc links * Add missing networkUser role. * Fix README.
GoogleCloudPlatform · Jun 28, 2023 · 0260712 · 0260712
1 parent bb1eaf5
commit 0260712
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 5 deletions.
diff --git a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf
@@ -84,7 +84,7 @@ module "processing-dp-historyserver" {
       staging_bucket = module.processing-staging-0.name
       temp_bucket    = module.processing-temp-0.name
       gce_cluster_config = {
-        subnetwork             = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
+        subnetwork             = local.processing_subnet
         zone                   = "${var.region}-b"
         service_account        = module.processing-sa-0.email
         service_account_scopes = ["cloud-platform"]

diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf
@@ -50,12 +50,12 @@ locals {
   processing_subnet = (
     local.use_shared_vpc
     ? var.network_config.subnet_self_link
-    : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"]
+    : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null)
   )
   processing_vpc = (
     local.use_shared_vpc
     ? var.network_config.network_self_link
-    : module.processing-vpc.0.self_link
+    : try(module.processing-vpc.0.self_link, null)
   )
 }
 
@@ -101,7 +101,7 @@ module "processing-project" {
     host_project = var.network_config.host_project
     service_identity_iam = {
       "roles/compute.networkUser" = [
-        "cloudservices", "compute", "container-engine", "dataflow"
+        "cloudservices", "compute", "container-engine", "dataflow", "dataproc"
       ]
       "roles/composer.sharedVpcAgent" = [
         "composer"

diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md
@@ -69,7 +69,7 @@ We use three groups to control access to resources:
 
 ### Virtual Private Cloud (VPC) design
 
-As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project.
+As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc.
 
 If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).