diff --git a/README.md b/README.md index 23a04c47f5..813dc26b28 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | @@ -192,7 +193,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -200,6 +201,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index dd4527af8e..2d57eef899 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -206,10 +206,10 @@ resource "google_container_cluster" "primary" { enabled = identity_service_config.value } } + {% endif %} enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting {% endif %} - {% endif %} {% if beta_cluster %} enable_fqdn_network_policy = var.enable_fqdn_network_policy {% endif %} @@ -239,6 +239,17 @@ resource "google_container_cluster" "primary" { } {% endif %} +{% if autopilot_cluster != true %} + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } +{% endif %} + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 009c181324..663a10661b 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -306,13 +306,12 @@ variable "node_pools_oauth_scopes" { } {% endif %} -{% if autopilot_cluster %} variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } -{% endif %} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -762,7 +761,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -783,6 +782,12 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } + +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} {% endif %} {% if beta_cluster %} {% if autopilot_cluster != true %} @@ -823,12 +828,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 3146875d47..5a156c7ccc 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -108,6 +108,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 0f21e99a4e..2eae51d4bd 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." diff --git a/cluster.tf b/cluster.tf index 2236446061..6c6f379c3b 100644 --- a/cluster.tf +++ b/cluster.tf @@ -147,6 +147,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -160,6 +162,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/examples/island_cluster_with_vm_router/README.md b/examples/island_cluster_with_vm_router/README.md new file mode 100644 index 0000000000..e36d8ed115 --- /dev/null +++ b/examples/island_cluster_with_vm_router/README.md @@ -0,0 +1,38 @@ +# GKE island cluster using VM as router + +This example provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project. + +1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network. +1. Outbound connections will go through the router. +1. For inbound connections, use Private Service Connect. + +## Deploy + +1. Update `project_id`, `cluster_name` and `primary_subnet` values in `terraform.tfvars`, and update other variables as needed. +1. Run `terraform apply`. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name | n/a | `string` | n/a | yes | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes | +| node\_locations | n/a | `list(string)` | n/a | yes | +| primary\_net\_cidrs | n/a | `list(string)` | n/a | yes | +| primary\_subnet | n/a | `string` | n/a | yes | +| project\_id | n/a | `string` | n/a | yes | +| proxy\_subnet\_cidr | n/a | `string` | n/a | yes | +| psc\_subnet\_cidr | n/a | `string` | n/a | yes | +| region | n/a | `string` | n/a | yes | +| router\_machine\_type | n/a | `string` | n/a | yes | +| secondary\_ranges | n/a | `map(string)` | n/a | yes | +| subnet\_cidr | n/a | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_id | n/a | + + diff --git a/examples/island_cluster_with_vm_router/main.tf b/examples/island_cluster_with_vm_router/main.tf new file mode 100644 index 0000000000..86ce138888 --- /dev/null +++ b/examples/island_cluster_with_vm_router/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "rand" { + byte_length = 4 +} + +resource "google_service_account" "gke-sa" { + account_id = "gke-sa-${random_id.rand.hex}" + project = var.project_id +} + +module "net" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + + network_name = "gke-net-${random_id.rand.hex}" + routing_mode = "GLOBAL" + project_id = var.project_id + delete_default_internet_gateway_routes = true + + subnets = [ + { + subnet_name = "${var.cluster_name}-${var.region}-snet" + subnet_ip = var.subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${var.cluster_name}-${var.region}-proxy-snet" + subnet_ip = var.proxy_subnet_cidr + subnet_region = var.region + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" + }, + { + subnet_name = "${var.cluster_name}-${var.region}-psc-snet" + subnet_ip = var.psc_subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + purpose = "PRIVATE_SERVICE_CONNECT" + } + ] + + secondary_ranges = { + "${var.cluster_name}-${var.region}-snet" = [ + { + range_name = "${var.cluster_name}-${var.region}-snet-pods" + ip_cidr_range = var.secondary_ranges["pods"] + }, + { + range_name = "${var.cluster_name}-${var.region}-snet-services" + ip_cidr_range = var.secondary_ranges["services"] + }, + ] + } + + routes = flatten([ + [for k, v in var.primary_net_cidrs : + { + name = "${var.cluster_name}-egress-gke-${k}" + description = "egress through the router for range ${v}" + destination_range = v + tags = "gke-${random_id.rand.hex}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ], + [ + { + name = "${var.cluster_name}-default-igw" + description = "internet through the router" + destination_range = "0.0.0.0/0" + tags = "gke-${random_id.rand.hex}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ] + ]) + + firewall_rules = [ + { + name = "${var.cluster_name}-iap" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + ports = ["22"] + } + ] + ranges = ["35.235.240.0/20"] + }, + { + name = "${var.cluster_name}-tcp-primary" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [ + var.subnet_cidr, + var.secondary_ranges["pods"] + ] + }, + { + name = "${var.cluster_name}-allow-psc" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [var.psc_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + }, + { + name = "${var.cluster_name}-allow-proxy" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [var.proxy_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + }, + ] +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" + version = "~> 30.0" + + depends_on = [google_compute_instance.vm] + + name = var.cluster_name + project_id = var.project_id + region = var.region + release_channel = "RAPID" + zones = var.node_locations + network = module.net.network_name + subnetwork = "${var.cluster_name}-${var.region}-snet" + ip_range_pods = "${var.cluster_name}-${var.region}-snet-pods" + ip_range_services = "${var.cluster_name}-${var.region}-snet-services" + enable_private_endpoint = true + enable_private_nodes = true + datapath_provider = "ADVANCED_DATAPATH" + monitoring_enable_managed_prometheus = false + enable_shielded_nodes = true + master_global_access_enabled = false + master_ipv4_cidr_block = var.secondary_ranges["master_cidr"] + master_authorized_networks = var.master_authorized_networks + deletion_protection = false + remove_default_node_pool = true + disable_default_snat = true + gateway_api_channel = "CHANNEL_STANDARD" + + node_pools = [ + { + name = "default" + machine_type = "e2-highcpu-2" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = true + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = google_service_account.gke-sa.email + initial_node_count = 1 + enable_secure_boot = true + }, + ] + + node_pools_tags = { + all = ["gke-${random_id.rand.hex}"] + } + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + } + + timeouts = { + create = "15m" + update = "15m" + delete = "15m" + } +} + +resource "google_gke_hub_membership" "primary" { + provider = google-beta + + project = var.project_id + membership_id = "${var.project_id}-${module.gke.name}" + location = var.region + + endpoint { + gke_cluster { + resource_link = "//container.googleapis.com/${module.gke.cluster_id}" + } + } + authority { + issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}" + } +} diff --git a/examples/island_cluster_with_vm_router/outputs.tf b/examples/island_cluster_with_vm_router/outputs.tf new file mode 100644 index 0000000000..640692ed75 --- /dev/null +++ b/examples/island_cluster_with_vm_router/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + value = module.gke.cluster_id +} diff --git a/examples/island_cluster_with_vm_router/router.tf b/examples/island_cluster_with_vm_router/router.tf new file mode 100644 index 0000000000..aee86f74cf --- /dev/null +++ b/examples/island_cluster_with_vm_router/router.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance" "vm" { + project = var.project_id + zone = var.node_locations[0] + name = "${var.cluster_name}-router-${random_id.rand.hex}" + machine_type = var.router_machine_type + allow_stopping_for_update = true + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + } + } + can_ip_forward = true + shielded_instance_config { + enable_secure_boot = true + } + network_interface { + subnetwork = var.primary_subnet + } + network_interface { + subnetwork = module.net.subnets["${var.region}/${var.cluster_name}-${var.region}-snet"]["self_link"] + } + metadata_startup_script = <<-EOT + #!/bin/bash + set -ex + sudo apt-get update + echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p + sudo iptables -A FORWARD -i ens5 -o ens4 -j ACCEPT + sudo iptables -A FORWARD -i ens4 -o ens5 -m state --state ESTABLISHED,RELATED -j ACCEPT + GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway" + GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google") + sudo ip route add ${var.secondary_ranges["pods"]} via $${GWY_IP} dev ens5 + sudo iptables -t nat -A POSTROUTING -o ens4 -s 0.0.0.0/0 -j MASQUERADE + EOT +} diff --git a/examples/island_cluster_with_vm_router/terraform.tfvars b/examples/island_cluster_with_vm_router/terraform.tfvars new file mode 100644 index 0000000000..3e9b74c54b --- /dev/null +++ b/examples/island_cluster_with_vm_router/terraform.tfvars @@ -0,0 +1,29 @@ +project_id = "" +region = "us-central1" +cluster_name = "gke-island-cluster-test" +node_locations = [ + "us-central1-a", + "us-central1-b", + "us-central1-f" +] +subnet_cidr = "100.64.0.0/20" +router_machine_type = "n2-highcpu-4" +primary_subnet = "projects//regions//subnetworks/" +secondary_ranges = { + pods = "100.64.64.0/18" + services = "100.64.128.0/20" + master_cidr = "100.64.144.0/28" +} +proxy_subnet_cidr = "100.64.168.0/24" +psc_subnet_cidr = "100.64.192.0/24" +master_authorized_networks = [ + { + cidr_block = "100.64.0.0/10" + display_name = "cluster net" + } +] +primary_net_cidrs = [ + "10.0.0.0/8", + "192.168.0.0/16", + "172.16.0.0/12" +] diff --git a/examples/island_cluster_with_vm_router/vars.tf b/examples/island_cluster_with_vm_router/vars.tf new file mode 100644 index 0000000000..37cf62d182 --- /dev/null +++ b/examples/island_cluster_with_vm_router/vars.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "node_locations" { + type = list(string) +} + +variable "subnet_cidr" { + type = string +} + +variable "psc_subnet_cidr" { + type = string +} + +variable "proxy_subnet_cidr" { + type = string +} + +variable "secondary_ranges" { + type = map(string) +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "primary_subnet" { + type = string +} + +variable "primary_net_cidrs" { + type = list(string) +} + +variable "router_machine_type" { + type = string +} diff --git a/examples/island_cluster_with_vm_router/versions.tf b/examples/island_cluster_with_vm_router/versions.tf new file mode 100644 index 0000000000..7e297ed82e --- /dev/null +++ b/examples/island_cluster_with_vm_router/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.6" + + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + } +} diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index e7fc994480..7f894a0c94 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -121,7 +121,7 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` |
[
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index b0af2e3623..397c7755b9 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -107,6 +107,7 @@ resource "google_container_cluster" "primary" { } } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index faa9d3d1f7..02d8e12e48 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -174,10 +174,11 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 81f6883bbd..7ff4ee1cdb 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -110,7 +110,7 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` |
[
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 43878c77b2..01ea54baf4 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -107,6 +107,7 @@ resource "google_container_cluster" "primary" { } } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 29a3db949b..fc10df626c 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -174,10 +174,11 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 9fc4f48564..1315894262 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -242,7 +242,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -250,6 +250,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index a153bb8663..5860bf18a0 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -185,6 +185,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d149edca13..03742b1504 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -724,7 +730,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -746,6 +752,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -782,12 +794,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index df855450e7..7bd4dde460 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -220,7 +220,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -228,6 +228,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 12807cbdc7..8e67389f51 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -185,6 +185,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d149edca13..03742b1504 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -724,7 +730,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -746,6 +752,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -782,12 +794,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 61d34f1e25..3918b354a1 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -231,7 +231,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -239,6 +239,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 1a1f44cb3d..eb30115d14 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -185,6 +185,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 40023d59b2..2490cfd338 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -694,7 +700,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -716,6 +722,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -752,12 +764,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 1019f74e73..37853e7e76 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -209,7 +209,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -217,6 +217,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index da18df1fc0..96879d12d7 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -185,6 +185,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 40023d59b2..2490cfd338 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -694,7 +700,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -716,6 +722,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -752,12 +764,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index ddd8cc56e6..d980cb858f 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -187,6 +187,7 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | @@ -225,7 +226,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -233,6 +234,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 84d63e5fbf..a579e512d0 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -147,6 +147,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -160,6 +162,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index e0884739d7..e7e2e0e5d0 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -700,7 +706,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -722,6 +728,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 91d314167f..9fb785d5b9 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -165,6 +165,7 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | @@ -203,7 +204,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -211,6 +212,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` |
[
{
"name": "default-node-pool"
}
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` |
{
"all": {},
"default-node-pool": {}
}
| no | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 7bb8f67db2..67ee8f1e4a 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -147,6 +147,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -160,6 +162,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index e0884739d7..e7e2e0e5d0 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -700,7 +706,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -722,6 +728,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 8aa383cec1..274199a59f 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -249,6 +249,7 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index 3da128f44e..f79239f47d 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -104,6 +104,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index 2b51a97541..c98da16f81 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 8aa383cec1..274199a59f 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -249,6 +249,7 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 67a3a54ed4..ac8b6d9409 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -104,6 +104,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 2b51a97541..c98da16f81 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." diff --git a/test/integration/go.mod b/test/integration/go.mod index 342f622d96..d5828f9c0a 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -5,8 +5,8 @@ go 1.21 toolchain go1.21.5 require ( - github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1 - github.com/gruntwork-io/terratest v0.46.11 + github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.2 + github.com/gruntwork-io/terratest v0.46.13 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 ) diff --git a/test/integration/go.sum b/test/integration/go.sum index 0549f453d9..5bafcdb72a 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -187,8 +187,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1 h1:8eKlk/DQeXPb6ITTLLWk/LdmyC9FRNMQF2ZR0sKaGEA= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1/go.mod h1:jIatwk/2sLSDtnMaExpzZpQVuBbEhx+NeiP1obo/IlY= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.2 h1:DQS5D0xaV1SVFNoGvHdYB0wYeZ24D9hVeFiLG2+r06I= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.2/go.mod h1:uomWciQn6oAotW86ck0b0ix+sUeZHSXyS+IYAVFTPj8= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -376,8 +376,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= -github.com/gruntwork-io/terratest v0.46.11 h1:1Z9G18I2FNuH87Ro0YtjW4NH9ky4GDpfzE7+ivkPeB8= -github.com/gruntwork-io/terratest v0.46.11/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= +github.com/gruntwork-io/terratest v0.46.13 h1:FDaEoZ7DtkomV8pcwLdBV/VsytdjnPRqJkIriYEYwjs= +github.com/gruntwork-io/terratest v0.46.13/go.mod h1:8sxu3Qup8TxtbzOHzq0MUrQffJj/G61/OwlsReaCwpo= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/variables.tf b/variables.tf index ddfbb6d741..1989020509 100644 --- a/variables.tf +++ b/variables.tf @@ -300,6 +300,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -670,7 +676,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -692,6 +698,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string