From aa8c6f6d95182ca87619417dc6fae7d8ca139d17 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 28 Jun 2023 14:10:16 +0200 Subject: [PATCH 1/4] Fix --- .../data-solutions/data-platform-minimal/02-processing.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf index 53da3fa683..57e02e001e 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf @@ -50,12 +50,12 @@ locals { processing_subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_link - : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"] + : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null) ) processing_vpc = ( local.use_shared_vpc ? var.network_config.network_self_link - : module.processing-vpc.0.self_link + : try(module.processing-vpc.0.self_link, null) ) } From ccf152706728f85b2c977427eafc9aa78fff4141 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 28 Jun 2023 15:25:16 +0200 Subject: [PATCH 2/4] Fix dataproc vpc links --- blueprints/data-solutions/data-platform-minimal/02-dataproc.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf index 4275c559a4..3a68a7a8f4 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf @@ -84,7 +84,7 @@ module "processing-dp-historyserver" { staging_bucket = module.processing-staging-0.name temp_bucket = module.processing-temp-0.name gce_cluster_config = { - subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link + subnetwork = local.processing_subnet zone = "${var.region}-b" service_account = module.processing-sa-0.email service_account_scopes = ["cloud-platform"] From 0190450660998a18e31df8648a5452223180e0e0 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 28 Jun 2023 15:52:33 +0200 Subject: [PATCH 3/4] Add missing networkUser role. --- .../data-solutions/data-platform-minimal/02-processing.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf index 57e02e001e..1d8cca2a05 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf @@ -101,7 +101,7 @@ module "processing-project" { host_project = var.network_config.host_project service_identity_iam = { "roles/compute.networkUser" = [ - "cloudservices", "compute", "container-engine", "dataflow" + "cloudservices", "compute", "container-engine", "dataflow", "dataproc" ] "roles/composer.sharedVpcAgent" = [ "composer" From 20a2538bbce70c45c781e96d3a0b923cd6213cad Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 28 Jun 2023 17:44:33 +0200 Subject: [PATCH 4/4] Fix README. --- blueprints/data-solutions/data-platform-minimal/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index e459c37fc0..3d00ea4939 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -69,7 +69,7 @@ We use three groups to control access to resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. +As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc. If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).