update to support autopilot and node_config_defaults

terraform-google-modules · Sep 13, 2024 · 0411c32 · 0411c32
1 parent f121604
commit 0411c32
Show file tree

Hide file tree

Showing 32 changed files with 209 additions and 43 deletions.
diff --git a/README.md b/README.md
@@ -189,6 +189,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -270,8 +270,12 @@ resource "google_container_cluster" "primary" {
   }
 {% if autopilot_cluster %}
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled !="" ? [1] : []
     content {
+      dynamic "kubelet_config" {
+        for_each = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
+          insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
+      }
       network_tags {
         tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }
@@ -660,7 +664,6 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
-  {% if beta_cluster %}
 
   node_pool_defaults {
     node_config_defaults {
@@ -677,12 +680,12 @@ resource "google_container_cluster" "primary" {
       gcfs_config {
         enabled = var.enable_gcfs
       }
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
       {% endif %}
     }
   }
-  {% endif %}
-  {% if beta_cluster %}
 
+  {% if beta_cluster %}
   depends_on = [google_project_iam_member.service_agent]
   {% endif %}
 }
@@ -1047,7 +1050,7 @@ resource "google_container_node_pool" "windows_pools" {
         cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
         cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
         cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
-        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }

diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -102,6 +102,17 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type = string
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = ""
+
+  validation {
+    condition     = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
+    error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
+  }
+}
+
 {% if autopilot_cluster != true %}
 variable "datapath_provider" {
   type        = string

diff --git a/cluster.tf b/cluster.tf
@@ -499,6 +499,16 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+
+  node_pool_defaults {
+    node_config_defaults {
+      gcfs_config {
+        enabled = var.enable_gcfs
+      }
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
+    }
+  }
+
 }
 /******************************************
   Create Container Cluster node pools
@@ -745,7 +755,7 @@ resource "google_container_node_pool" "pools" {
         cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
         cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
         cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
-        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
@@ -1028,7 +1038,7 @@ resource "google_container_node_pool" "windows_pools" {
         cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
         cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
         cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
-        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -79,7 +79,7 @@ module "gke" {
       sandbox_enabled                        = true
       cpu_manager_policy                     = "static"
       cpu_cfs_quota                          = true
-      insecure_kubelet_readonly_port_enabled = "TRUE"
+      insecure_kubelet_readonly_port_enabled = "FALSE"
       local_ssd_ephemeral_count              = 2
       pod_pids_limit                         = 4096
     },

diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf
@@ -61,11 +61,12 @@ module "gke" {
 
   node_pools = [
     {
-      name            = "pool-01"
-      min_count       = 1
-      max_count       = 2
-      service_account = var.compute_engine_service_account
-      auto_upgrade    = true
+      name                                   = "pool-01"
+      min_count                              = 1
+      max_count                              = 2
+      service_account                        = var.compute_engine_service_account
+      auto_upgrade                           = true
+      insecure_kubelet_readonly_port_enabled = "FALSE"
     },
     {
       name              = "pool-02"

diff --git a/examples/simple_autopilot_public/main.tf b/examples/simple_autopilot_public/main.tf
@@ -36,22 +36,23 @@ module "gke" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-public-cluster"
   version = "~> 33.0"
 
-  project_id                      = var.project_id
-  name                            = "${local.cluster_type}-cluster"
-  regional                        = true
-  region                          = var.region
-  network                         = module.gcp-network.network_name
-  subnetwork                      = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
-  ip_range_pods                   = local.pods_range_name
-  ip_range_services               = local.svc_range_name
-  release_channel                 = "RAPID"
-  enable_vertical_pod_autoscaling = true
-  network_tags                    = [local.cluster_type]
-  deletion_protection             = false
-  enable_l4_ilb_subsetting        = true
-  gcs_fuse_csi_driver             = true
-  stateful_ha                     = false
-  gke_backup_agent_config         = false
+  project_id                             = var.project_id
+  name                                   = "${local.cluster_type}-cluster"
+  regional                               = true
+  region                                 = var.region
+  network                                = module.gcp-network.network_name
+  subnetwork                             = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                          = local.pods_range_name
+  ip_range_services                      = local.svc_range_name
+  release_channel                        = "RAPID"
+  enable_vertical_pod_autoscaling        = true
+  network_tags                           = [local.cluster_type]
+  deletion_protection                    = false
+  enable_l4_ilb_subsetting               = true
+  gcs_fuse_csi_driver                    = true
+  insecure_kubelet_readonly_port_enabled = "FALSE"
+  stateful_ha                            = false
+  gke_backup_agent_config                = false
   ray_operator_config = {
     enabled            = true
     logging_enabled    = true

diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md
@@ -113,6 +113,7 @@ Then perform the following commands on the root folder:
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf
@@ -118,8 +118,12 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
     content {
+      dynamic "kubelet_config" {
+        for_each                               = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
+        insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
+      }
       network_tags {
         tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }

diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf
@@ -102,6 +102,17 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = string
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = ""
+
+  validation {
+    condition     = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
+    error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
+  }
+}
+
 variable "maintenance_start_time" {
   type        = string
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"

diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md
@@ -104,6 +104,7 @@ Then perform the following commands on the root folder:
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf
@@ -118,8 +118,12 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
     content {
+      dynamic "kubelet_config" {
+        for_each                               = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
+        insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
+      }
       network_tags {
         tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }

diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf
@@ -102,6 +102,17 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = string
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = ""
+
+  validation {
+    condition     = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
+    error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
+  }
+}
+
 variable "maintenance_start_time" {
   type        = string
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
@@ -232,6 +232,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -580,6 +580,7 @@ resource "google_container_cluster" "primary" {
       gcfs_config {
         enabled = var.enable_gcfs
       }
+      insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
     }
   }
 
@@ -918,7 +919,7 @@ resource "google_container_node_pool" "pools" {
         cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
         cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
         cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
-        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }
@@ -1215,7 +1216,7 @@ resource "google_container_node_pool" "windows_pools" {
         cpu_manager_policy                     = lookup(each.value, "cpu_manager_policy", "static")
         cpu_cfs_quota                          = lookup(each.value, "cpu_cfs_quota", null)
         cpu_cfs_quota_period                   = lookup(each.value, "cpu_cfs_quota_period", null)
-        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
+        insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
         pod_pids_limit                         = lookup(each.value, "pod_pids_limit", null)
       }
     }

diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
@@ -102,6 +102,17 @@ variable "service_external_ips" {
   default     = false
 }
 
+variable "insecure_kubelet_readonly_port_enabled" {
+  type        = string
+  description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
+  default     = ""
+
+  validation {
+    condition     = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
+    error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
+  }
+}
+
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
@@ -210,6 +210,7 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
 | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
+| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |