From 90e9bdfb2a348ab22d996711e477cc6a7aa27c28 Mon Sep 17 00:00:00 2001
From: Till Adam <lxmail@web.de>
Date: Mon, 6 Nov 2023 17:42:04 +0100
Subject: [PATCH 1/3] feat: add advanced datapath observability config option
 (#1776)

---
 README.md                                        |  2 ++
 autogen/main/cluster.tf.tmpl                     |  4 ++++
 autogen/main/variables.tf.tmpl                   | 16 ++++++++++++++++
 cluster.tf                                       |  4 ++++
 .../README.md                                    |  2 ++
 .../cluster.tf                                   |  4 ++++
 .../variables.tf                                 | 16 ++++++++++++++++
 modules/beta-private-cluster/README.md           |  2 ++
 modules/beta-private-cluster/cluster.tf          |  4 ++++
 modules/beta-private-cluster/variables.tf        | 16 ++++++++++++++++
 .../beta-public-cluster-update-variant/README.md |  2 ++
 .../cluster.tf                                   |  4 ++++
 .../variables.tf                                 | 16 ++++++++++++++++
 modules/beta-public-cluster/README.md            |  2 ++
 modules/beta-public-cluster/cluster.tf           |  4 ++++
 modules/beta-public-cluster/variables.tf         | 16 ++++++++++++++++
 modules/private-cluster-update-variant/README.md |  2 ++
 .../private-cluster-update-variant/cluster.tf    |  4 ++++
 .../private-cluster-update-variant/variables.tf  | 16 ++++++++++++++++
 modules/private-cluster/README.md                |  2 ++
 modules/private-cluster/cluster.tf               |  4 ++++
 modules/private-cluster/variables.tf             | 16 ++++++++++++++++
 variables.tf                                     | 16 ++++++++++++++++
 23 files changed, 174 insertions(+)

diff --git a/README.md b/README.md
index ee07f25fc3..8668b49981 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,9 @@ Then perform the following commands on the root folder:
 | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
 | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
index b4a78f53bd..4d36e19941 100644
--- a/autogen/main/cluster.tf.tmpl
+++ b/autogen/main/cluster.tf.tmpl
@@ -125,6 +125,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
index bbc76e46e1..b4937103d9 100644
--- a/autogen/main/variables.tf.tmpl
+++ b/autogen/main/variables.tf.tmpl
@@ -739,6 +739,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/cluster.tf b/cluster.tf
index a8d6f4a389..2332cc8e26 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -90,6 +90,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 3133216c38..ce955dc88a 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -239,7 +239,9 @@ Then perform the following commands on the root folder:
 | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no |
 | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 87edc410f1..c07a41aa1c 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -102,6 +102,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 69cd142d8f..b4e79f3c33 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -699,6 +699,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 8664b78ef7..5e6b8f4d70 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -217,7 +217,9 @@ Then perform the following commands on the root folder:
 | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no |
 | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 048c452336..830e70a93e 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -102,6 +102,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index 69cd142d8f..b4e79f3c33 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -699,6 +699,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md
index b9e5f81369..1596ee9f62 100644
--- a/modules/beta-public-cluster-update-variant/README.md
+++ b/modules/beta-public-cluster-update-variant/README.md
@@ -228,7 +228,9 @@ Then perform the following commands on the root folder:
 | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
 | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
index c0fe6d89dc..d0671e557f 100644
--- a/modules/beta-public-cluster-update-variant/cluster.tf
+++ b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -102,6 +102,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf
index 461e0e4f4d..653adb3516 100644
--- a/modules/beta-public-cluster-update-variant/variables.tf
+++ b/modules/beta-public-cluster-update-variant/variables.tf
@@ -669,6 +669,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 94b8f8be5c..ddd7dd4c76 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -206,7 +206,9 @@ Then perform the following commands on the root folder:
 | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
 | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 5f2b3615e1..f757f769df 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -102,6 +102,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 461e0e4f4d..653adb3516 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -669,6 +669,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index e17f5dc79d..608f8a120d 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -221,7 +221,9 @@ Then perform the following commands on the root folder:
 | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no |
 | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index 8d47a9ddb4..aa9cd45ea9 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -90,6 +90,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 81b91e3b32..c593f7b3ed 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -663,6 +663,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 5c50f77b9d..cbc62c36dc 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -199,7 +199,9 @@ Then perform the following commands on the root folder:
 | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no |
 | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no |
 | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
+| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no |
 | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
+| monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
 | name | The name of the cluster (required) | `string` | n/a | yes |
 | network | The VPC network to host the cluster in (required) | `string` | n/a | yes |
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index d14d8b9701..f9c3266a6b 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -90,6 +90,10 @@ resource "google_container_cluster" "primary" {
       managed_prometheus {
         enabled = var.monitoring_enable_managed_prometheus
       }
+      advanced_datapath_observability_config {
+        enable_metrics = var.monitoring_enable_observability_metrics
+        relay_mode     = var.monitoring_observability_metrics_relay_mode
+      }
     }
   }
   cluster_autoscaling {
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
index 81b91e3b32..c593f7b3ed 100644
--- a/modules/private-cluster/variables.tf
+++ b/modules/private-cluster/variables.tf
@@ -663,6 +663,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
diff --git a/variables.tf b/variables.tf
index 13c6ba8047..a67db04b38 100644
--- a/variables.tf
+++ b/variables.tf
@@ -633,6 +633,22 @@ variable "monitoring_enable_managed_prometheus" {
   default     = false
 }
 
+variable "monitoring_enable_observability_metrics" {
+  type        = bool
+  description = "Whether or not the advanced datapath metrics are enabled."
+  default     = false
+}
+
+variable "monitoring_observability_metrics_relay_mode" {
+  type        = string
+  description = "Mode used to make advanced datapath metrics relay available."
+  default     = null
+  validation {
+    condition     = var.monitoring_observability_metrics_relay_mode == null ? true : contains(["DISABLED", "INTERNAL_VPC_LB", "EXTERNAL_LB"], var.monitoring_observability_metrics_relay_mode)
+    error_message = "The advanced datapath metrics relay value must be one of DISABLED, INTERNAL_VPC_LB, EXTERNAL_LB."
+  }
+}
+
 variable "monitoring_enabled_components" {
   type        = list(string)
   description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."

From f4a1a3ab606a3c69ec4b85ae41e62d46cc7a8ed5 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Fri, 10 Nov 2023 13:05:42 -0800
Subject: [PATCH 2/3] chore(deps): Update GO modules (#1797)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
---
 test/integration/go.mod | 4 ++--
 test/integration/go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/integration/go.mod b/test/integration/go.mod
index 23f1c0e8a5..407bda3726 100644
--- a/test/integration/go.mod
+++ b/test/integration/go.mod
@@ -3,8 +3,8 @@ module github.com/terraform-google-modules/terraform-google-kubernetes-engine/te
 go 1.20
 
 require (
-	github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2
-	github.com/gruntwork-io/terratest v0.46.1
+	github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.10.0
+	github.com/gruntwork-io/terratest v0.46.5
 	github.com/stretchr/testify v1.8.4
 )
 
diff --git a/test/integration/go.sum b/test/integration/go.sum
index e61a1a3cf1..025efdf3b7 100644
--- a/test/integration/go.sum
+++ b/test/integration/go.sum
@@ -187,8 +187,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2 h1:7fdp02N9fd8itrSe/p7njaSKAUYJGgxn8ajgZfbFK+I=
-github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2/go.mod h1:yyde2qkA+GhCou8exSJwifnJlAcWCNcU1vs911CEOJg=
+github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.10.0 h1:z3SfgfmUGWlmYPRl4YsEqyEexAYzPNos65401mppP+o=
+github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.10.0/go.mod h1:yyde2qkA+GhCou8exSJwifnJlAcWCNcU1vs911CEOJg=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
@@ -370,8 +370,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0=
 github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk=
-github.com/gruntwork-io/terratest v0.46.1 h1:dJ/y2/Li6yCDIc8KXY8PfydtrMRiXFb3UZm4LoPShPI=
-github.com/gruntwork-io/terratest v0.46.1/go.mod h1:gl//tb5cLnbpQs1FTSNwhsrbhsoG00goCJPfOnyliiU=
+github.com/gruntwork-io/terratest v0.46.5 h1:cmsIAKjM1Hqwy5tlZPb6EJQvaMCD4xRX1DN9fnTptBM=
+github.com/gruntwork-io/terratest v0.46.5/go.mod h1:6gI5MlLeyF+SLwqocA5GBzcTix+XiuxCy1BPwKuT+WM=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

From a9de2d79d68b19dfee5b6bd8ee0c646ee621bbee Mon Sep 17 00:00:00 2001
From: Marco Ferrari <ferrarim@google.com>
Date: Sat, 11 Nov 2023 01:41:02 +0200
Subject: [PATCH 3/3] feat(TF>=1.1)!: Configure ASM management mode (#1702)

Co-authored-by: Andrew Peabody <andrewpeabody@google.com>
---
 modules/asm/README.md    |  4 +++-
 modules/asm/hub.tf       | 12 +++++++++++
 modules/asm/main.tf      | 45 +++++++++++++++++++++++++++++++++++++++-
 modules/asm/outputs.tf   |  2 +-
 modules/asm/variables.tf | 17 ++++++++++++++-
 modules/asm/versions.tf  |  2 +-
 6 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/modules/asm/README.md b/modules/asm/README.md
index 85480fcf3c..782626b309 100644
--- a/modules/asm/README.md
+++ b/modules/asm/README.md
@@ -34,6 +34,7 @@ module "asm" {
 ```
 
 Note that the [`mesh_id` label on the cluster](https://cloud.google.com/service-mesh/docs/managed/auto-control-plane-with-fleet#apply_the_mesh_id_label) is required for metrics to get displayed on the Anthos Service Mesh pages in the Cloud console (Topology, etc.). Illustrated with the full example mentioned above, here is an example of what your cluster should have:
+
 ```tf
 module "gke" {
 ...
@@ -58,9 +59,10 @@ To deploy this config:
 | enable\_cni | Determines whether to enable CNI for this ASM installation. Required to use Managed Data Plane (MDP). | `bool` | `false` | no |
 | enable\_fleet\_registration | Determines whether the module registers the cluster to the fleet. | `bool` | `false` | no |
 | enable\_mesh\_feature | Determines whether the module enables the mesh feature on the fleet. | `bool` | `false` | no |
-| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc | `bool` | `false` | no |
+| enable\_vpc\_sc | Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc) | `bool` | `false` | no |
 | fleet\_id | The fleet to use for this ASM installation. | `string` | `""` | no |
 | internal\_ip | Use internal ip for the cluster endpoint when running kubectl commands. | `bool` | `false` | no |
+| mesh\_management | ASM Management mode. For more information, see the [gke\_hub\_feature\_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh) | `string` | `""` | no |
 | module\_depends\_on | List of modules or resources this module depends on.  If multiple, all items must be the same type. | `list(any)` | `[]` | no |
 | multicluster\_mode | [Preview] Determines whether remote secrets should be autogenerated across fleet cluster. | `string` | `"manual"` | no |
 | project\_id | The project in which the resource belongs. | `string` | n/a | yes |
diff --git a/modules/asm/hub.tf b/modules/asm/hub.tf
index 62840bbcfd..a10f044b5d 100644
--- a/modules/asm/hub.tf
+++ b/modules/asm/hub.tf
@@ -33,3 +33,15 @@ resource "google_gke_hub_feature" "mesh" {
   location = "global"
   provider = google-beta
 }
+
+resource "google_gke_hub_feature_membership" "mesh_feature_membership" {
+  count = var.enable_fleet_registration && var.enable_mesh_feature && var.mesh_management != "" ? 1 : 0
+
+  location   = "global"
+  feature    = google_gke_hub_feature.mesh[0].name
+  membership = google_gke_hub_membership.membership[0].membership_id
+  mesh {
+    management = var.mesh_management
+  }
+  provider = google-beta
+}
diff --git a/modules/asm/main.tf b/modules/asm/main.tf
index 58c3292081..666621c816 100644
--- a/modules/asm/main.tf
+++ b/modules/asm/main.tf
@@ -34,7 +34,7 @@ data "google_container_cluster" "asm" {
 }
 
 resource "kubernetes_namespace" "system" {
-  count = var.create_system_namespace ? 1 : 0
+  count = var.create_system_namespace && var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0
 
   metadata {
     name = "istio-system"
@@ -42,6 +42,8 @@ resource "kubernetes_namespace" "system" {
 }
 
 resource "kubernetes_config_map" "asm_options" {
+  count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0
+
   metadata {
     name      = "asm-options"
     namespace = try(kubernetes_namespace.system[0].metadata[0].name, "istio-system")
@@ -56,6 +58,8 @@ resource "kubernetes_config_map" "asm_options" {
 }
 
 module "cpr" {
+  count = var.mesh_management != "MANAGEMENT_AUTOMATIC" ? 1 : 0
+
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version = "~> 3.1"
 
@@ -69,3 +73,42 @@ module "cpr" {
 
   module_depends_on = [kubernetes_config_map.asm_options]
 }
+
+# Wait for the ControlPlaneRevision custom resource to be ready.
+# Add an explicit "retry until the resource is created" until
+module "kubectl_asm_wait_for_controlplanerevision_custom_resource_definition" {
+  count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0
+
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version = "~> 3.1"
+
+  project_id              = var.project_id
+  cluster_name            = var.cluster_name
+  cluster_location        = var.cluster_location
+  kubectl_create_command  = "/bin/sh -c 'while ! kubectl wait crd/controlplanerevisions.mesh.cloud.google.com --for condition=established --timeout=60m --all-namespaces; do echo \"crd/controlplanerevisions.mesh.cloud.google.com not yet available, waiting...\"; sleep 5; done'"
+  kubectl_destroy_command = ""
+
+  module_depends_on = [
+    google_gke_hub_feature_membership.mesh_feature_membership
+  ]
+}
+
+# Wait for the ASM control plane revision to be ready so we can safely deploy resources that depend
+# on ASM mutating webhooks.
+# Add an explicit "retry until the resource is created" until
+module "kubectl_asm_wait_for_controlplanerevision" {
+  count = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? 1 : 0
+
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version = "~> 3.1"
+
+  project_id              = var.project_id
+  cluster_name            = var.cluster_name
+  cluster_location        = var.cluster_location
+  kubectl_create_command  = "/bin/sh -c 'while ! kubectl -n istio-system wait ControlPlaneRevision --all --timeout=60m --for condition=Reconciled; do echo \"ControlPlaneRevision not yet available, waiting...\"; sleep 5; done'"
+  kubectl_destroy_command = ""
+
+  module_depends_on = [
+    module.kubectl_asm_wait_for_controlplanerevision_custom_resource_definition[0].wait
+  ]
+}
diff --git a/modules/asm/outputs.tf b/modules/asm/outputs.tf
index ba9237238c..936f148ff2 100644
--- a/modules/asm/outputs.tf
+++ b/modules/asm/outputs.tf
@@ -20,6 +20,6 @@ output "revision_name" {
 }
 
 output "wait" {
-  value       = module.cpr.wait
+  value       = var.mesh_management == "MANAGEMENT_AUTOMATIC" ? module.kubectl_asm_wait_for_controlplanerevision[0].wait : module.cpr[0].wait
   description = "An output to use when depending on the ASM installation finishing."
 }
diff --git a/modules/asm/variables.tf b/modules/asm/variables.tf
index 92d2869c4c..f5a0cb6941 100644
--- a/modules/asm/variables.tf
+++ b/modules/asm/variables.tf
@@ -50,6 +50,21 @@ variable "channel" {
   default = ""
 }
 
+variable "mesh_management" {
+  default     = ""
+  description = "ASM Management mode. For more information, see the [gke_hub_feature_membership resource documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#nested_mesh)"
+  type        = string
+  validation {
+    condition = anytrue([
+      var.mesh_management == null,
+      var.mesh_management == "",
+      var.mesh_management == "MANAGEMENT_AUTOMATIC",
+      var.mesh_management == "MANAGEMENT_MANUAL",
+    ])
+    error_message = "Must be null, empty, or one of MANAGEMENT_AUTOMATIC or MANAGEMENT_MANUAL."
+  }
+}
+
 variable "multicluster_mode" {
   description = "[Preview] Determines whether remote secrets should be autogenerated across fleet cluster."
   type        = string
@@ -70,7 +85,7 @@ variable "enable_cni" {
 }
 
 variable "enable_vpc_sc" {
-  description = "Determines whether to enable VPC-SC for this ASM installation. For more information read https://cloud.google.com/service-mesh/docs/managed/vpc-sc"
+  description = "Determines whether to enable VPC-SC for this ASM installation. For more information read [VPC Service Controls for Managed Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/managed/vpc-sc)"
   type        = bool
   default     = false
 }
diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf
index 1947e780cb..958bab4728 100644
--- a/modules/asm/versions.tf
+++ b/modules/asm/versions.tf
@@ -16,7 +16,7 @@
  */
 
 terraform {
-  required_version = ">= 0.14.0"
+  required_version = ">= 1.1"
 
   required_providers {
     kubernetes = {