From 7cd9083fd7ab47e46757a4945b5b65b188d0d02d Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:28:28 +0100 Subject: [PATCH 1/8] data-platform-minimal - support web_server_network_access_control --- .../data-solutions/data-platform-minimal/02-composer.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 218df0445b..09b3d85974 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -109,6 +109,15 @@ resource "google_composer_environment" "processing-cmp-0" { kms_key_name = var.service_encryption_keys.composer } } + web_server_network_access_control { + dynamic "allowed_ip_range" { + for_each = var.webserver_access_ip_ranges + content { + value = allowed_ip_range.value["value"] + description = allowed_ip_range.value["description"] + } + } + } } depends_on = [ module.processing-project From b30efa8bfe7c06e5ace85ee8d2580a7ec3374929 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:56:00 +0100 Subject: [PATCH 2/8] Update variables.tf --- .../data-solutions/data-platform-minimal/variables.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index 09cdfdb8f3..40692b76a2 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,6 +24,16 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) + web_server_network_access_ip_ranges = optional(list(map(string)), [ + { + description = "Allows access from all IPv4 addresses (default value)" + value = "0.0.0.0/0" + }, + { + description = "Allows access from all IPv6 addresses (default value)" + value = "::0/0" + } + ]) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) @@ -106,7 +116,6 @@ variable "network_config" { pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) - # web_server_network_access_control = list(string) }) nullable = false default = {} From 9e8323bd7d6a7f4acc5a07f7b0b8d68189516fa2 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:56:35 +0100 Subject: [PATCH 3/8] Update 02-composer.tf --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 09b3d85974..4b6dbaffdb 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.webserver_access_ip_ranges + for_each = var.composer_config.web_server_network_access_ip_ranges content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] From f9742c5405f1f6293af0d7b5b4b1eb0d691c1c9a Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 14:06:05 +0100 Subject: [PATCH 4/8] lint --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 4b6dbaffdb..9ddb8a7be2 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -113,7 +113,7 @@ resource "google_composer_environment" "processing-cmp-0" { dynamic "allowed_ip_range" { for_each = var.composer_config.web_server_network_access_ip_ranges content { - value = allowed_ip_range.value["value"] + value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] } } From 52ad1fee34d0b68e334ac3c99eb0247e7c40ef81 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 14:07:36 +0100 Subject: [PATCH 5/8] more linting --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 9ddb8a7be2..f4360226f0 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.composer_config.web_server_network_access_ip_ranges + for_each = var.composer_config.web_server_network_access_ip_ranges content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] From e3e05540debc9bc765599dc01010a5b890bf0073 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:45:29 +0100 Subject: [PATCH 6/8] change var name, drop default, update docs --- .../data-platform-minimal/02-composer.tf | 2 +- .../data-solutions/data-platform-minimal/README.md | 14 +++++++------- .../data-platform-minimal/variables.tf | 11 +---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index f4360226f0..a890064ba4 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.composer_config.web_server_network_access_ip_ranges + for_each = var.composer_config.web_server_access_control content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index e459c37fc0..f6ac475ca5 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -280,13 +280,13 @@ The application layer is out of scope of this script. As a demo purpuse only, on | [organization_domain](variables.tf#L119) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L124) | Prefix used for resource names. | string | ✓ | | | [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | -| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | -| [groups](variables.tf#L80) | User groups. | map(string) | | {…} | -| [location](variables.tf#L90) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | +| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | +| [groups](variables.tf#L81) | User groups. | map(string) | | {…} | +| [location](variables.tf#L91) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L97) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | | [project_suffix](variables.tf#L157) | Suffix used only for project ids. | string | | null | | [region](variables.tf#L163) | Region used for regional resources. | string | | "europe-west1" | | [service_encryption_keys](variables.tf#L169) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | {} | diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index 40692b76a2..afe5738324 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,16 +24,7 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) - web_server_network_access_ip_ranges = optional(list(map(string)), [ - { - description = "Allows access from all IPv4 addresses (default value)" - value = "0.0.0.0/0" - }, - { - description = "Allows access from all IPv6 addresses (default value)" - value = "::0/0" - } - ]) + web_server_access_control = optional(map(string)) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) From f15e2f35092fbe5777e0b34b83db09d2f81fdac9 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:55:10 +0100 Subject: [PATCH 7/8] default to empty map --- blueprints/data-solutions/data-platform-minimal/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index afe5738324..a5f5143ea8 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,7 +24,7 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) - web_server_access_control = optional(map(string)) + web_server_access_control = optional(map(string), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) From 912497a158bb21b8ae559ae92e972c4ca26f5878 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:57:23 +0100 Subject: [PATCH 8/8] use key/value and update docs --- .../data-solutions/data-platform-minimal/02-composer.tf | 4 ++-- blueprints/data-solutions/data-platform-minimal/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index a890064ba4..da6fca9ac1 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -113,8 +113,8 @@ resource "google_composer_environment" "processing-cmp-0" { dynamic "allowed_ip_range" { for_each = var.composer_config.web_server_access_control content { - value = allowed_ip_range.value["value"] - description = allowed_ip_range.value["description"] + value = allowed_ip_range.key + description = allowed_ip_range.value } } } diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index f6ac475ca5..5411349ba8 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -280,7 +280,7 @@ The application layer is out of scope of this script. As a demo purpuse only, on | [organization_domain](variables.tf#L119) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L124) | Prefix used for resource names. | string | ✓ | | | [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | | [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | | [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} |