From 272b5d97102960463e2314628b7933fc58120b39 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 4 Aug 2023 11:19:46 +0000 Subject: [PATCH 1/8] Configure ASM management mode --- modules/asm/README.md | 4 +++- modules/asm/hub.tf | 12 ++++++++++++ modules/asm/variables.tf | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/modules/asm/README.md b/modules/asm/README.md index 85480fcf3c..95c9d99110 100644 --- a/modules/asm/README.md +++ b/modules/asm/README.md @@ -34,6 +34,7 @@ module "asm" { ``` Note that the [`mesh_id` label on the cluster](https://cloud.google.com/service-mesh/docs/managed/auto-control-plane-with-fleet#apply_the_mesh_id_label) is required for metrics to get displayed on the Anthos Service Mesh pages in the Cloud console (Topology, etc.). Illustrated with the full example mentioned above, here is an example of what your cluster should have: + ```tf module "gke" { ... @@ -58,9 +59,10 @@ To deploy this config: | enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no | | enable\_fleet\_registration | Determines whether the module registers the cluster to the fleet. | `bool` | `false` | no | | enable\_mesh\_feature | Determines whether the module enables the mesh feature on the fleet. | `bool` | `false` | no | -| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc | `bool` | `false` | no | +| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read | `bool` | `false` | no | | fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no | | internal\_ip | Use internal ip for the cluster endpoint when running kubectl commands. | `bool` | `false` | no | +| mesh\_management | ASM management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `""` | no | | module\_depends\_on | List of modules or resources this module depends on. If multiple, all items must be the same type. | `list(any)` | `[]` | no | | multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no | | project\_id | The project in which the resource belongs. | `string` | n/a | yes | diff --git a/modules/asm/hub.tf b/modules/asm/hub.tf index 62840bbcfd..a10f044b5d 100644 --- a/modules/asm/hub.tf +++ b/modules/asm/hub.tf @@ -33,3 +33,15 @@ resource "google_gke_hub_feature" "mesh" { location = "global" provider = google-beta } + +resource "google_gke_hub_feature_membership" "mesh_feature_membership" { + count = var.enable_fleet_registration && var.enable_mesh_feature && var.mesh_management != "" ? 1 : 0 + + location = "global" + feature = google_gke_hub_feature.mesh[0].name + membership = google_gke_hub_membership.membership[0].membership_id + mesh { + management = var.mesh_management + } + provider = google-beta +} diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf index 92d2869c4c..f4d8c5cb34 100644 --- a/modules/asm/variables.tf +++ b/modules/asm/variables.tf @@ -50,6 +50,20 @@ variable "channel" { default = "" } +variable "mesh_management" { + default = "" + description = "ASM Management mode." + type = string + validation { + condition = anytrue([ + var.mesh_management == "", + var.mesh_management == "MANAGEMENT_AUTOMATIC", + var.mesh_management == "MANAGEMENT_MANUAL", + ]) + error_message = "Must be empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL." + } +} + variable "multicluster_mode" { description = "[Preview] Determines whether remote secrets should be autogenerated across fleet cluster." type = string From f298bfc7958f809462e0e4ad423fb83bb7ea0a67 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 29 Sep 2023 08:53:19 +0000 Subject: [PATCH 2/8] Don't create resources that are automatically managed --- modules/asm/README.md | 4 ++-- modules/asm/main.tf | 6 +++++- modules/asm/variables.tf | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/asm/README.md b/modules/asm/README.md index 95c9d99110..782626b309 100644 --- a/modules/asm/README.md +++ b/modules/asm/README.md @@ -59,10 +59,10 @@ To deploy this config: | enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no | | enable\_fleet\_registration | Determines whether the module registers the cluster to the fleet. | `bool` | `false` | no | | enable\_mesh\_feature | Determines whether the module enables the mesh feature on the fleet. | `bool` | `false` | no | -| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read | `bool` | `false` | no | +| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc) | `bool` | `false` | no | | fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no | | internal\_ip | Use internal ip for the cluster endpoint when running kubectl commands. | `bool` | `false` | no | -| mesh\_management | ASM management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `""` | no | +| mesh\_management | ASM Management mode. For more information, see the [gke\_hub\_feature\_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `""` | no | | module\_depends\_on | List of modules or resources this module depends on. If multiple, all items must be the same type. | `list(any)` | `[]` | no | | multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no | | project\_id | The project in which the resource belongs. | `string` | n/a | yes | diff --git a/modules/asm/main.tf b/modules/asm/main.tf index 58c3292081..2cc0b7c75f 100644 --- a/modules/asm/main.tf +++ b/modules/asm/main.tf @@ -34,7 +34,7 @@ data "google_container_cluster" "asm" { } resource "kubernetes_namespace" "system" { - count = var.create_system_namespace ? 1 : 0 + count = var.create_system_namespace && var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 metadata { name = "istio-system" @@ -42,6 +42,8 @@ resource "kubernetes_namespace" "system" { } resource "kubernetes_config_map" "asm_options" { + count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 + metadata { name = "asm-options" namespace = try(kubernetes_namespace.system[0].metadata[0].name, "istio-system") @@ -56,6 +58,8 @@ resource "kubernetes_config_map" "asm_options" { } module "cpr" { + count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0 + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" version = "~> 3.1" diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf index f4d8c5cb34..b04a783dcd 100644 --- a/modules/asm/variables.tf +++ b/modules/asm/variables.tf @@ -52,7 +52,7 @@ variable "channel" { variable "mesh_management" { default = "" - description = "ASM Management mode." + description = "ASM Management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh)" type = string validation { condition = anytrue([ @@ -84,7 +84,7 @@ variable "enable_cni" { } variable "enable_vpc_sc" { - description = "Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc" + description = "Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc)" type = bool default = false } From 4620333b16a1926cda20ee89db82ca0bd38502f9 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 3 Oct 2023 12:00:29 +0000 Subject: [PATCH 3/8] Don't set the wait output variable with auto management --- modules/asm/outputs.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf index ba9237238c..b0fa63f107 100644 --- a/modules/asm/outputs.tf +++ b/modules/asm/outputs.tf @@ -20,6 +20,6 @@ output "revision_name" { } output "wait" { - value = module.cpr.wait - description = "An output to use when depending on the ASM installation finishing." + value = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? module.cpr[0].wait : null + description = "An output to use when depending on the ASM installation finishing when NOT using automated ASM management modes." } From 69e176c219b1ff73a5b81555ae814fd0d72467e5 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 3 Oct 2023 12:07:27 +0000 Subject: [PATCH 4/8] Wait for ASM to be ready in auto mode --- modules/asm/main.tf | 39 +++++++++++++++++++++++++++++++++++++++ modules/asm/outputs.tf | 4 ++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/modules/asm/main.tf b/modules/asm/main.tf index 2cc0b7c75f..7d832fd787 100644 --- a/modules/asm/main.tf +++ b/modules/asm/main.tf @@ -73,3 +73,42 @@ module "cpr" { module_depends_on = [kubernetes_config_map.asm_options] } + +# Wait for the ControlPlaneRevision custom resource to be ready. +# Add an explicit "retry until the resource is created" until +module "kubectl_asm_wait_for_controlplanerevision_custom_resource_definition" { + count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0 + + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + project_id = var.project_id + cluster_name = var.cluster_name + cluster_location = var.cluster_location + kubectl_create_command = "/bin/sh -c 'while ! kubectl wait crd/controlplanerevisions.mesh.cloud.google.com --for condition=established --timeout=60m --all-namespaces; do echo \"crd/controlplanerevisions.mesh.cloud.google.com not yet available, waiting...\"; sleep 5; done'" + kubectl_destroy_command = "" + + module_depends_on = [ + google_gke_hub_feature_membership.membership + ] +} + +# Wait for the ASM control plane revision to be ready so we can safely deploy resources that depend +# on ASM mutating webhooks. +# Add an explicit "retry until the resource is created" until +module "kubectl_asm_wait_for_controlplanerevision" { + count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0 + + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + project_id = var.project_id + cluster_name = var.cluster_name + cluster_location = var.cluster_location + kubectl_create_command = "/bin/sh -c 'while ! kubectl -n istio-system wait ControlPlaneRevision --all --timeout=60m --for condition=Reconciled; do echo \"ControlPlaneRevision not yet available, waiting...\"; sleep 5; done'" + kubectl_destroy_command = "" + + module_depends_on = [ + module.kubectl_asm_wait_for_controlplanerevision_custom_resource_definition.wait + ] +} diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf index b0fa63f107..936f148ff2 100644 --- a/modules/asm/outputs.tf +++ b/modules/asm/outputs.tf @@ -20,6 +20,6 @@ output "revision_name" { } output "wait" { - value = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? module.cpr[0].wait : null - description = "An output to use when depending on the ASM installation finishing when NOT using automated ASM management modes." + value = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? module.kubectl_asm_wait_for_controlplanerevision[0].wait : module.cpr[0].wait + description = "An output to use when depending on the ASM installation finishing." } From bf69b06c68d1e2dd7ad3bb8cc102b34d7b763781 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 3 Oct 2023 12:22:08 +0000 Subject: [PATCH 5/8] Fix depends --- modules/asm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/asm/main.tf b/modules/asm/main.tf index 7d832fd787..cf9f677e9f 100644 --- a/modules/asm/main.tf +++ b/modules/asm/main.tf @@ -89,7 +89,7 @@ module "kubectl_asm_wait_for_controlplanerevision_custom_resource_definition" { kubectl_destroy_command = "" module_depends_on = [ - google_gke_hub_feature_membership.membership + google_gke_hub_feature_membership.mesh_feature_membership ] } From 0f6f501b86df8051e74526b8cf528a104fd0a563 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 3 Oct 2023 12:34:05 +0000 Subject: [PATCH 6/8] Fix depends --- modules/asm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/asm/main.tf b/modules/asm/main.tf index cf9f677e9f..666621c816 100644 --- a/modules/asm/main.tf +++ b/modules/asm/main.tf @@ -109,6 +109,6 @@ module "kubectl_asm_wait_for_controlplanerevision" { kubectl_destroy_command = "" module_depends_on = [ - module.kubectl_asm_wait_for_controlplanerevision_custom_resource_definition.wait + module.kubectl_asm_wait_for_controlplanerevision_custom_resource_definition[0].wait ] } From b5e10fbc0d5a2ac0b5407d5ebfa0161f978f02ee Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 4 Oct 2023 08:03:13 +0000 Subject: [PATCH 7/8] Allow null values --- modules/asm/variables.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf index b04a783dcd..f5a0cb6941 100644 --- a/modules/asm/variables.tf +++ b/modules/asm/variables.tf @@ -56,11 +56,12 @@ variable "mesh_management" { type = string validation { condition = anytrue([ + var.mesh_management == null, var.mesh_management == "", var.mesh_management == "MANAGEMENT_AUTOMATIC", var.mesh_management == "MANAGEMENT_MANUAL", ]) - error_message = "Must be empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL." + error_message = "Must be null, empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL." } } From da4df2bcc65ab5e3c7358bc9ec82315e22995a71 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 4 Oct 2023 08:04:13 +0000 Subject: [PATCH 8/8] Update min Terraform version to handle null values --- modules/asm/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index f628eb7d1d..99c41900cc 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -16,7 +16,7 @@ */ terraform { - required_version = ">= 0.14.0" + required_version = ">= 1.1" required_providers { kubernetes = {