Merge branch 'terraform-google-modules:master' into enable_blue_green…

…_upgrade_strategy

.github/workflows/stale.yml

@@ -21,7 +21,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v7
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days'

Makefile

@@ -18,8 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-# Pin to 1.3.9 per https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/issues/1208
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.8
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.10
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 DOCKER_BIN ?= docker

README.md

@@ -207,7 +207,6 @@ Then perform the following commands on the root folder:
 | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no |
 | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | <pre>object({<br>    metadata = string<br>  })</pre> | <pre>{<br>  "metadata": "INCLUDE_ALL_METADATA"<br>}</pre> | no |
 | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes |
 | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no |

autogen/main/cluster.tf.tmpl

@@ -135,12 +135,12 @@ resource "google_container_cluster" "primary" {
       content {
         service_account = local.service_account
         oauth_scopes = local.node_pools_oauth_scopes["all"]
-        
+
         management {
             auto_repair  = lookup(var.cluster_autoscaling, "auto_repair", true)
             auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade",true)
         }
-        
+
         {% if beta_cluster %}
         min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "")
         {% endif %}
@@ -309,7 +309,7 @@ resource "google_container_cluster" "primary" {
     {% endif %}
   }
   {% if autopilot_cluster != true %}
-  
+
   datapath_provider = var.datapath_provider
   {% endif %}
 

autogen/main/main.tf.tmpl

@@ -68,7 +68,7 @@ locals {
     resource_type = "memory"
     minimum       = var.cluster_autoscaling.min_memory_gb
     maximum       = var.cluster_autoscaling.max_memory_gb
-  }], var.cluster_autoscaling.gpu_resources) : []  
+  }], var.cluster_autoscaling.gpu_resources) : []
   {% endif %}
 
 
@@ -77,12 +77,14 @@ locals {
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
   zone_count                  = length(var.zones)
   cluster_type                = var.regional ? "regional" : "zonal"
+{% if autopilot_cluster != true %}
   // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous.
 {% if beta_cluster %}
   // When a release channel is used, node auto-upgrade are enabled and cannot be disabled.
   default_auto_upgrade = var.regional || var.release_channel != null ? true : false
 {% else %}
   default_auto_upgrade = var.regional ? true : false
+{% endif %}
 {% endif %}
 
   cluster_subnet_cidr       = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
@@ -133,14 +135,12 @@ locals {
   }]
   {% endif %}
 
-  cluster_output_name           = google_container_cluster.primary.name
   cluster_output_regional_zones = google_container_cluster.primary.node_locations
-  cluster_output_zonal_zones    = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : []
   cluster_output_zones          = local.cluster_output_regional_zones
 
 {% if private_cluster %}
-  cluster_endpoint           = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint
-  cluster_peering_name       = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null
+  cluster_endpoint           = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config[0].private_endpoint : google_container_cluster.primary.private_cluster_config[0].public_endpoint) : google_container_cluster.primary.endpoint
+  cluster_peering_name       = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config[0].peering_name : null
   cluster_endpoint_for_nodes = var.master_ipv4_cidr_block
 {% else %}
   cluster_endpoint           = google_container_cluster.primary.endpoint
@@ -152,17 +152,18 @@ locals {
   cluster_output_min_master_version                 = google_container_cluster.primary.min_master_version
   cluster_output_logging_service                    = google_container_cluster.primary.logging_service
   cluster_output_monitoring_service                 = google_container_cluster.primary.monitoring_service
-  cluster_output_network_policy_enabled             = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled
-  cluster_output_http_load_balancing_enabled        = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled
-  cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled
-  cluster_output_vertical_pod_autoscaling_enabled   = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false
+{% if autopilot_cluster != true %}
+  cluster_output_network_policy_enabled             = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled
+{% endif %}
+  cluster_output_http_load_balancing_enabled        = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled
+  cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled
+  cluster_output_vertical_pod_autoscaling_enabled   = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false
 
 {% if beta_cluster %}
   # BETA features
-  cluster_output_istio_disabled                   = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false
-  cluster_output_pod_security_policy_enabled      = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false
+  cluster_output_istio_disabled                   = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false
+  cluster_output_pod_security_policy_enabled      = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false
   cluster_output_intranode_visbility_enabled      = google_container_cluster.primary.enable_intranode_visibility
-  cluster_output_identity_service_enabled         = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config.0.enabled : false
 
   # /BETA features
   {% endif %}
@@ -176,11 +177,11 @@ locals {
    [for np in google_container_node_pool.pools : np.name], [""],
    [for np in google_container_node_pool.windows_pools : np.name], [""]
   )
-  
+
   cluster_output_node_pools_versions = merge(
     { for np in google_container_node_pool.pools : np.name => np.version },
     { for np in google_container_node_pool.windows_pools : np.name => np.version },
-  )  
+  )
   {% endif %}
 
   cluster_master_auth_list_layer1 = local.cluster_output_master_auth
@@ -221,7 +222,9 @@ locals {
   # BETA features
   cluster_istio_enabled                    = ! local.cluster_output_istio_disabled
   cluster_dns_cache_enabled                = var.dns_cache
+  {% if autopilot_cluster != true %}
   cluster_telemetry_type_is_set            = var.cluster_telemetry_type != null
+  {% endif %}
   cluster_pod_security_policy_enabled      = local.cluster_output_pod_security_policy_enabled
   cluster_intranode_visibility_enabled     = local.cluster_output_intranode_visbility_enabled
   confidential_node_config                 = var.enable_confidential_nodes == true ? [{ enabled = true }] : []

autogen/main/variables.tf.tmpl

@@ -102,12 +102,14 @@ variable "service_external_ips" {
   default     = false
 }
 
+{% if autopilot_cluster != true %}
 variable "datapath_provider" {
   type        = string
   description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."
   default     = "DATAPATH_PROVIDER_UNSPECIFIED"
 }
 
+{% endif %}
 variable "maintenance_start_time" {
   type        = string
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"
@@ -335,14 +337,15 @@ variable "configure_ip_masq" {
   default     = false
 }
 
-{% if beta_cluster %}
+{% if beta_cluster and autopilot_cluster != true%}
 variable "cluster_telemetry_type" {
   type        = string
   description = "Available options include ENABLED, DISABLED, and SYSTEM_ONLY"
   default     = null
 }
 
 {% endif %}
+{% if autopilot_cluster != true%}
 variable "logging_service" {
   type        = string
   description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none"
@@ -355,6 +358,7 @@ variable "monitoring_service" {
   default     = "monitoring.googleapis.com/kubernetes"
 }
 
+{% endif %}
 variable "create_service_account" {
   type        = bool
   description = "Defines if service account specified to run nodes should be created."
@@ -397,11 +401,6 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
-variable "skip_provisioners" {
-  type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
-  default     = false
-}
 {% if private_cluster %}
 
 variable "deploy_using_private_endpoint" {
@@ -697,6 +696,7 @@ variable "enable_kubernetes_alpha" {
 
 variable "istio" {
   description = "(Beta) Enable Istio addon"
+  type        = bool
   default     = false
 }
 
@@ -720,12 +720,14 @@ variable "config_connector" {
 
 variable "cloudrun" {
   description = "(Beta) Enable CloudRun addon"
+  type        = bool
   default     = false
 }
 
 variable "cloudrun_load_balancer_type" {
   description = "(Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer."
-  default = ""
+  type        = string
+  default     = ""
 }
 
 variable "enable_pod_security_policy" {

autogen/main/versions.tf.tmpl

@@ -22,6 +22,10 @@ terraform {
 
 {% if beta_cluster %}
   required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.51.0, < 5.0"
+    }
     google-beta = {
       source  = "hashicorp/google-beta"
       version = ">= 4.51.0, < 5.0"
@@ -30,6 +34,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "~> 2.10"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 2.1"
+    }
   }
   provider_meta "google-beta" {
     module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v25.0.0"
@@ -44,6 +52,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "~> 2.10"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 2.1"
+    }
   }
   provider_meta "google" {
     module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v25.0.0"

autogen/safer-cluster/main.tf.tmpl

@@ -26,6 +26,7 @@ module "gke" {
   {% endif %}
   project_id         = var.project_id
   name               = var.name
+  description        = var.description
   regional           = var.regional
   region             = var.region
   zones              = var.zones
@@ -188,8 +189,6 @@ module "gke" {
 
   enable_shielded_nodes = var.enable_shielded_nodes
 
-  skip_provisioners = var.skip_provisioners
-
   gce_pd_csi_driver    = var.gce_pd_csi_driver
   filestore_csi_driver = var.filestore_csi_driver
 

autogen/safer-cluster/variables.tf.tmpl

@@ -306,6 +306,7 @@ variable "master_ipv4_cidr_block" {
 
 variable "istio" {
   description = "(Beta) Enable Istio addon"
+  type        = bool
   default     = false
 }
 
@@ -341,6 +342,7 @@ variable "cluster_dns_domain" {
 
 variable "default_max_pods_per_node" {
   description = "The maximum number of pods to schedule per node"
+  type        = number
   default     = 110
 }
 
@@ -355,6 +357,7 @@ variable "database_encryption" {
 
 variable "cloudrun" {
   description = "(Beta) Enable CloudRun addon"
+  type        = bool
   default     = false
 }
 
@@ -364,18 +367,6 @@ variable "resource_usage_export_dataset_id" {
   default     = ""
 }
 
-variable "enable_network_egress_export" {
-  type        = bool
-  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
-  default     = false
-}
-
-variable "enable_resource_consumption_export" {
-  type        = bool
-  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
-  default     = true
-}
-
 variable "enable_cost_allocation" {
   type        = bool
   description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery"
@@ -424,12 +415,6 @@ variable "enable_private_endpoint" {
   default     = true
 }
 
-variable "skip_provisioners" {
-  type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
-  default     = false
-}
-
 variable "enable_pod_security_policy" {
   type        = bool
   description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."

build/int.cloudbuild.yaml

@@ -511,6 +511,6 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.8'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.10'
 options:
   machineType: 'N1_HIGHCPU_8'

build/lint.cloudbuild.yaml

@@ -22,7 +22,7 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.8'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.10'
 options:
   machineType: 'N1_HIGHCPU_8'
   env:

examples/simple_autopilot_private_non_default_sa/main.tf

@@ -48,7 +48,6 @@ module "gke" {
   enable_private_endpoint         = true
   enable_private_nodes            = true
   master_ipv4_cidr_block          = "172.16.0.0/28"
-  datapath_provider               = "ADVANCED_DATAPATH"
 
   master_authorized_networks = [
     {

examples/simple_regional/README.md

@@ -15,7 +15,6 @@ This example illustrates how to create a simple cluster.
 | network | The VPC network to host the cluster in | `any` | n/a | yes |
 | project\_id | The project ID to host the cluster in | `any` | n/a | yes |
 | region | The region to host the cluster in | `any` | n/a | yes |
-| skip\_provisioners | Flag to skip local-exec provisioners | `bool` | `false` | no |
 | subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes |
 
 ## Outputs

examples/simple_regional/main.tf

@@ -40,5 +40,4 @@ module "gke" {
   service_account             = var.compute_engine_service_account
   enable_cost_allocation      = true
   enable_binary_authorization = var.enable_binary_authorization
-  skip_provisioners           = var.skip_provisioners
 }

examples/simple_regional/variables.tf

@@ -47,12 +47,6 @@ variable "compute_engine_service_account" {
   description = "Service account to associate to the nodes in the cluster"
 }
 
-variable "skip_provisioners" {
-  type        = bool
-  description = "Flag to skip local-exec provisioners"
-  default     = false
-}
-
 variable "enable_binary_authorization" {
   description = "Enable BinAuthZ Admission controller"
   default     = false

examples/simple_regional_beta/README.md

@@ -10,7 +10,6 @@ This example illustrates how to create a simple cluster with beta features.
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes |
 | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | <pre>[<br>  {<br>    "key_name": "",<br>    "state": "DECRYPTED"<br>  }<br>]</pre> | no |
-| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no |
 | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no |
 | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no |

examples/simple_regional_beta/variables.tf

@@ -113,9 +113,3 @@ variable "regional" {
   description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)"
   default     = true
 }
-
-variable "datapath_provider" {
-  type        = string
-  description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."
-  default     = "DATAPATH_PROVIDER_UNSPECIFIED"
-}