diff --git a/README.md b/README.md index e6a8945f55..581cf77bc3 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ Then perform the following commands on the root folder: | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -257,6 +258,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index e04d86406d..bec9779ea5 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -189,10 +189,9 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility - enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index 90efd57d37..73c44d1ddd 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -57,7 +57,6 @@ resource "google_compute_firewall" "intra_egress" { } -{% if beta_cluster %} /****************************************** Allow egress to the TPU IPv4 CIDR block @@ -95,8 +94,6 @@ resource "google_compute_firewall" "tpu_egress" { {% endif %} } - -{% endif %} /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 832054f9cc..eb84418487 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -171,6 +171,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + {% if autopilot_cluster != true %} output "mesh_certificates_config" { description = "Mesh certificates configuration" @@ -228,9 +233,4 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} {% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 7445f9d476..682ea65300 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -600,13 +600,12 @@ variable "deletion_protection" { default = true } -{% if beta_cluster %} variable "enable_tpu" { type = bool description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } -{% endif %} + {% if autopilot_cluster != true %} variable "network_policy" { type = bool diff --git a/cluster.tf b/cluster.tf index 2228fe8bbf..be4e3cbc3f 100644 --- a/cluster.tf +++ b/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/firewall.tf b/firewall.tf index 94cec9e103..b99cecd357 100644 --- a/firewall.tf +++ b/firewall.tf @@ -55,6 +55,41 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-private-cluster/firewall.tf b/modules/beta-autopilot-private-cluster/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-autopilot-private-cluster/firewall.tf +++ b/modules/beta-autopilot-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index 0d955524ae..8ba200ea1c 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "master_ipv4_cidr_block" { @@ -183,8 +188,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 373fa527a2..2a24cc62dd 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -433,6 +433,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-autopilot-public-cluster/firewall.tf b/modules/beta-autopilot-public-cluster/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-autopilot-public-cluster/firewall.tf +++ b/modules/beta-autopilot-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index e33c8aafa6..e60b61f903 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "cloudrun_enabled" { @@ -173,8 +178,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 2c5679d00b..caf41cda82 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -403,6 +403,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 434bfedbb7..9aaf838470 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 72aee4055d..64465ccde9 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -209,8 +214,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 5c18f14727..753d7b93bb 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 1c8e68452c..cc2ccb3999 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 72aee4055d..64465ccde9 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -209,8 +214,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 5c18f14727..753d7b93bb 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index a79bf757d0..fcc48cf172 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index bb7cdcc72c..5d199a4ed4 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -199,8 +204,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index e6f3eab0e4..5863324c52 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 57f7d04c18..dddf6c018e 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index bb7cdcc72c..5d199a4ed4 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -199,8 +204,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e6f3eab0e4..5863324c52 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 646278d5ca..9407ab5fb6 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -193,6 +193,7 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -292,6 +293,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 2c97d719c6..91f632c958 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index a5d89cefe0..3908a63364 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 722e3b8fd1..5b6a461d7a 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 483f3f3f48..3d493dc4a7 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -549,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index f6f4851284..54e5080862 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -171,6 +171,7 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -270,6 +271,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index b410c17958..7bacc938d8 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index a5d89cefe0..3908a63364 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 722e3b8fd1..5b6a461d7a 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 483f3f3f48..3d493dc4a7 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -549,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/outputs.tf b/outputs.tf index 04bbc957fe..232d0b1f29 100644 --- a/outputs.tf +++ b/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/variables.tf b/variables.tf index ea04722045..0b47543630 100644 --- a/variables.tf +++ b/variables.tf @@ -519,6 +519,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon"