From 05487dff3a37e5e9c7318a91e7b8b9905510e59c Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 29 Apr 2023 15:32:00 +0200 Subject: [PATCH 1/3] shorten stage 3 prefixes, enforce prefix length in stage 3s --- .../data-platform-foundations/03-orchestration.tf | 14 +++++++++----- fast/stages/0-bootstrap/variables.tf | 1 - fast/stages/3-data-platform/dev/main.tf | 2 +- fast/stages/3-data-platform/dev/variables.tf | 4 ++++ fast/stages/3-gke-multitenant/dev/main.tf | 2 +- fast/stages/3-gke-multitenant/dev/variables.tf | 4 ++++ 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index 799981be28..40752288f7 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -75,10 +75,14 @@ module "orch-project" { billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null - oslogin = false + name = ( + var.project_config.billing_account_id == null + ? var.project_config.project_ids.orc + : "${var.project_config.project_ids.orc}${local.project_suffix}" + ) + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", @@ -194,7 +198,7 @@ module "orch-sa-df-build" { prefix = var.prefix name = "orc-sa-df-build" display_name = "Data platform Dataflow build service account" - # Note values below should pertain to the system / group / users who are able to + # Note values below should pertain to the system / group / users who are able to # invoke the build via this service account iam = { "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] diff --git a/fast/stages/0-bootstrap/variables.tf b/fast/stages/0-bootstrap/variables.tf index 7aec437c7a..c85822892c 100644 --- a/fast/stages/0-bootstrap/variables.tf +++ b/fast/stages/0-bootstrap/variables.tf @@ -209,7 +209,6 @@ variable "outputs_location" { variable "prefix" { description = "Prefix used for resources that need unique names. Use 9 characters or less." type = string - validation { condition = try(length(var.prefix), 0) < 10 error_message = "Use a maximum of 9 characters for prefix." diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf index c600a7586f..6e699837b5 100644 --- a/fast/stages/3-data-platform/dev/main.tf +++ b/fast/stages/3-data-platform/dev/main.tf @@ -46,7 +46,7 @@ module "data-platform" { } } organization_domain = var.organization.domain - prefix = "${var.prefix}-dev-data" + prefix = "${var.prefix}-dev-dt" project_services = var.project_services project_suffix = "0" region = var.region diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index 74a5dbe11d..1a04011d57 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -167,6 +167,10 @@ variable "prefix" { # tfdoc:variable:source 00-globals description = "Unique prefix used for resource names. Not used for projects if 'project_create' is null." type = string + validation { + condition = try(length(var.prefix), 0) < 13 + error_message = "Use a maximum of 12 characters for prefix." + } } variable "project_services" { diff --git a/fast/stages/3-gke-multitenant/dev/main.tf b/fast/stages/3-gke-multitenant/dev/main.tf index 8d87b9076e..0fd29f5a1e 100644 --- a/fast/stages/3-gke-multitenant/dev/main.tf +++ b/fast/stages/3-gke-multitenant/dev/main.tf @@ -20,7 +20,7 @@ module "gke-multitenant" { source = "../../../../blueprints/gke/multitenant-fleet" billing_account_id = var.billing_account.id folder_id = var.folder_ids.gke-dev - project_id = "gke-clusters-0" + project_id = "gke-0" group_iam = var.group_iam iam = var.iam labels = merge(var.labels, { environment = "dev" }) diff --git a/fast/stages/3-gke-multitenant/dev/variables.tf b/fast/stages/3-gke-multitenant/dev/variables.tf index 2dbf5a6ea5..db532dd0eb 100644 --- a/fast/stages/3-gke-multitenant/dev/variables.tf +++ b/fast/stages/3-gke-multitenant/dev/variables.tf @@ -217,6 +217,10 @@ variable "outputs_location" { variable "prefix" { description = "Prefix used for resources that need unique names." type = string + validation { + condition = try(length(var.prefix), 0) < 13 + error_message = "Use a maximum of 12 characters for prefix." + } } variable "project_services" { From 79a3088d60a357312ba513e6329f411eb80fd16f Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 2 May 2023 10:58:01 +0200 Subject: [PATCH 2/3] tfdoc --- fast/stages/0-bootstrap/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md index eba8ec2e40..f784651f6e 100644 --- a/fast/stages/0-bootstrap/README.md +++ b/fast/stages/0-bootstrap/README.md @@ -515,7 +515,7 @@ The remaining configuration is manual, as it regards the repositories themselves | [locations](variables.tf#L150) | Optional locations for GCS, BigQuery, and logging buckets created here. | object({…}) | | {…} | | | [log_sinks](variables.tf#L169) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | | [outputs_location](variables.tf#L203) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable. | string | | null | | -| [project_parent_ids](variables.tf#L219) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | | +| [project_parent_ids](variables.tf#L218) | Optional parents for projects created here in folders/nnnnnnn format. Null values will use the organization as parent. | object({…}) | | {…} | | ## Outputs From f404af141c4fcea920a596724d3c9770f38677d2 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 2 May 2023 11:00:49 +0200 Subject: [PATCH 3/3] tfdoc --- fast/stages/3-data-platform/dev/README.md | 10 +++++----- fast/stages/3-gke-multitenant/dev/README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 48d09eafc0..07a8ad4661 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -197,11 +197,11 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | | [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | | [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [project_services](variables.tf#L176) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L187) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L193) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L205) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L214) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs diff --git a/fast/stages/3-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md index f0460c06c5..3cc4e33045 100644 --- a/fast/stages/3-gke-multitenant/dev/README.md +++ b/fast/stages/3-gke-multitenant/dev/README.md @@ -167,7 +167,7 @@ Leave all these variables unset (or set to `null`) to disable fleet management. | [folder_ids](variables.tf#L153) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman | | [host_project_ids](variables.tf#L168) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking | | [prefix](variables.tf#L217) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L229) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | +| [vpc_self_links](variables.tf#L233) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | | [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | | | [fleet_configmanagement_clusters](variables.tf#L90) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | | [fleet_configmanagement_templates](variables.tf#L98) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | @@ -178,7 +178,7 @@ Leave all these variables unset (or set to `null`) to disable fleet management. | [labels](variables.tf#L183) | Project-level labels. | map(string) | | {} | | | [nodepools](variables.tf#L189) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | | [outputs_location](variables.tf#L211) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L222) | Additional project services to enable. | list(string) | | [] | | +| [project_services](variables.tf#L226) | Additional project services to enable. | list(string) | | [] | | ## Outputs