From de8fe2a9023513f5fb48a7d8ea16d8da5f636bdc Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 25 May 2023 15:28:16 -0700
Subject: [PATCH 1/7] chore(deps): update tf modules (#1624)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
---
 examples/acm-terraform-blog-part1/terraform/gke.tf | 2 +-
 examples/acm-terraform-blog-part2/terraform/gke.tf | 2 +-
 examples/acm-terraform-blog-part3/terraform/gke.tf | 4 ++--
 examples/safer_cluster_iap_bastion/network.tf      | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/acm-terraform-blog-part1/terraform/gke.tf b/examples/acm-terraform-blog-part1/terraform/gke.tf
index 9d6ce2868c..d877dae6a0 100644
--- a/examples/acm-terraform-blog-part1/terraform/gke.tf
+++ b/examples/acm-terraform-blog-part1/terraform/gke.tf
@@ -31,7 +31,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google"
-  version            = "~> 25.0"
+  version            = "~> 26.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part1"
   region             = var.region
diff --git a/examples/acm-terraform-blog-part2/terraform/gke.tf b/examples/acm-terraform-blog-part2/terraform/gke.tf
index a8b94387c1..ceee1ffd0e 100644
--- a/examples/acm-terraform-blog-part2/terraform/gke.tf
+++ b/examples/acm-terraform-blog-part2/terraform/gke.tf
@@ -31,7 +31,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google"
-  version            = "~> 25.0"
+  version            = "~> 26.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part2"
   region             = var.region
diff --git a/examples/acm-terraform-blog-part3/terraform/gke.tf b/examples/acm-terraform-blog-part3/terraform/gke.tf
index b412f36da7..dc338f29aa 100644
--- a/examples/acm-terraform-blog-part3/terraform/gke.tf
+++ b/examples/acm-terraform-blog-part3/terraform/gke.tf
@@ -33,7 +33,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster"
-  version            = "~> 25.0"
+  version            = "~> 26.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part3"
   region             = var.region
@@ -48,7 +48,7 @@ module "gke" {
 
 module "wi" {
   source              = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
-  version             = "~> 25.0"
+  version             = "~> 26.0"
   gcp_sa_name         = "cnrmsa"
   cluster_name        = module.gke.name
   name                = "cnrm-controller-manager"
diff --git a/examples/safer_cluster_iap_bastion/network.tf b/examples/safer_cluster_iap_bastion/network.tf
index 13638f1105..10a952660a 100644
--- a/examples/safer_cluster_iap_bastion/network.tf
+++ b/examples/safer_cluster_iap_bastion/network.tf
@@ -49,7 +49,7 @@ module "vpc" {
 
 module "cloud-nat" {
   source        = "terraform-google-modules/cloud-nat/google"
-  version       = "~> 2.0"
+  version       = "~> 3.0"
   project_id    = module.enabled_google_apis.project_id
   region        = var.region
   router        = "safer-router"

From 9e3d16a7e36c2d6f730e9b31a138906941a4a674 Mon Sep 17 00:00:00 2001
From: Andrew Peabody <andrewpeabody@google.com>
Date: Thu, 1 Jun 2023 08:55:22 -0700
Subject: [PATCH 2/7] chore: expose enable_fleet_feature in ACM example (#1648)

---
 examples/simple_zonal_with_acm/README.md    | 1 +
 examples/simple_zonal_with_acm/acm.tf       | 2 ++
 examples/simple_zonal_with_acm/variables.tf | 6 ++++++
 3 files changed, 9 insertions(+)

diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md
index 9c8c4fe2e5..986b66e3cd 100644
--- a/examples/simple_zonal_with_acm/README.md
+++ b/examples/simple_zonal_with_acm/README.md
@@ -39,6 +39,7 @@ After applying the Terraform configuration, you can run the following commands t
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
+| enable\_fleet\_feature | Whether to enable the ACM feature on the fleet. | `bool` | `true` | no |
 | project\_id | The project ID to host the cluster in | `string` | n/a | yes |
 | region | The region to host the cluster in | `string` | `"us-central1"` | no |
 | zone | The zone to host the cluster in | `string` | `"us-central1-a"` | no |
diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf
index c72df471e0..e4662d08f9 100644
--- a/examples/simple_zonal_with_acm/acm.tf
+++ b/examples/simple_zonal_with_acm/acm.tf
@@ -24,6 +24,8 @@ module "acm" {
   sync_branch = "1.0.0"
   policy_dir  = "foo-corp"
 
+  enable_fleet_feature = var.enable_fleet_feature
+
   secret_type = "ssh"
 
   policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"]
diff --git a/examples/simple_zonal_with_acm/variables.tf b/examples/simple_zonal_with_acm/variables.tf
index 722b32e674..7885098dad 100644
--- a/examples/simple_zonal_with_acm/variables.tf
+++ b/examples/simple_zonal_with_acm/variables.tf
@@ -36,3 +36,9 @@ variable "zone" {
   description = "The zone to host the cluster in"
   default     = "us-central1-a"
 }
+
+variable "enable_fleet_feature" {
+  description = "Whether to enable the ACM feature on the fleet."
+  type        = bool
+  default     = true
+}

From 7a0f4caef75bc6a272a41e2250263ad6224bf587 Mon Sep 17 00:00:00 2001
From: CFT Bot <cloud-foundation-bot@google.com>
Date: Thu, 1 Jun 2023 10:16:51 -0700
Subject: [PATCH 3/7] chore: update .github/workflows/lint.yaml

---
 .github/workflows/lint.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index cf1bae4f34..fedc554dd6 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -35,7 +35,7 @@ jobs:
       - id: variables
         run: |
           MAKEFILE=$(find . -name Makefile -print -quit)
-          if [ ! -z "$MAKEFILE" ]; then
+          if [ -z "$MAKEFILE" ]; then
             echo dev-tools=gcr.io/cloud-foundation-cicd/cft/developer-tools:1 >> "$GITHUB_OUTPUT"
           else
             VERSION=$(grep "DOCKER_TAG_VERSION_DEVELOPER_TOOLS := " $MAKEFILE | cut -d\  -f3)

From 044968d3393ddfc3b3710b16187016151f06c4fb Mon Sep 17 00:00:00 2001
From: CFT Bot <cloud-foundation-bot@google.com>
Date: Thu, 1 Jun 2023 12:18:59 -0700
Subject: [PATCH 4/7] chore: update .github/renovate.json

---
 .github/renovate.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/renovate.json b/.github/renovate.json
index bccd0c174a..bb2b478857 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -16,6 +16,9 @@
      "stabilityDays":0
   },
   "separateMajorMinor":false,
+  "constraints": {
+    "go": "1.18"
+  },
   "packageRules": [
     {
       "matchPaths": ["examples/**", "test/**", ".github/**"],
@@ -35,7 +38,8 @@
       "postUpdateOptions": ["gomodTidy"]
     },
     {
-      "matchPackageNames": ["go"],
+      "matchDatasources": ["golang-version"],
+      "rangeStrategy": "bump",
       "allowedVersions": "<1.19.0",
       "postUpdateOptions": ["gomodTidy"]
     },

From e51804ed4849fa85748a95ca169b92445d258d91 Mon Sep 17 00:00:00 2001
From: Andrew Peabody <andrewpeabody@google.com>
Date: Fri, 2 Jun 2023 09:01:33 -0700
Subject: [PATCH 5/7] fix: policy bundles now use idx as resource name (bundles
 will be re-applied) (#1657)

---
 modules/acm/creds.tf          | 4 ++--
 modules/acm/policy_bundles.tf | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/acm/creds.tf b/modules/acm/creds.tf
index fb8b5fb379..5bd395da97 100644
--- a/modules/acm/creds.tf
+++ b/modules/acm/creds.tf
@@ -30,12 +30,12 @@ resource "tls_private_key" "k8sop_creds" {
   rsa_bits  = 4096
 }
 
-# Wait for the ACM operator to create the namespace
+# Wait for ACM
 resource "time_sleep" "wait_acm" {
   count      = (var.create_ssh_key == true || var.ssh_auth_key != null || var.enable_policy_controller || var.enable_config_sync) ? 1 : 0
   depends_on = [google_gke_hub_feature_membership.main]
 
-  create_duration = "300s"
+  create_duration = (length(var.policy_bundles) > 0) ? "600s" : "300s"
 }
 
 resource "google_service_account_iam_binding" "ksa_iam" {
diff --git a/modules/acm/policy_bundles.tf b/modules/acm/policy_bundles.tf
index b03987e867..8bd122d1e0 100644
--- a/modules/acm/policy_bundles.tf
+++ b/modules/acm/policy_bundles.tf
@@ -18,12 +18,13 @@ module "policy_bundles" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version = "~> 3.1"
 
-  for_each                = toset(var.policy_bundles)
+  # Use index as name to avoid long url or special filesystem chars
+  for_each                = { for i, v in var.policy_bundles : i => v }
   project_id              = var.project_id
   cluster_name            = var.cluster_name
   cluster_location        = var.location
-  kubectl_create_command  = "kubectl apply -k ${each.key}"
-  kubectl_destroy_command = "kubectl delete -k ${each.key}"
+  kubectl_create_command  = "kubectl apply -k ${each.value}"
+  kubectl_destroy_command = "kubectl delete -k ${each.value}"
 
   module_depends_on = [time_sleep.wait_acm]
 }

From e522073f24067359f8af1bd2ddc9092b594fb945 Mon Sep 17 00:00:00 2001
From: slimatic <salim.31@gmail.com>
Date: Fri, 2 Jun 2023 12:59:15 -0400
Subject: [PATCH 6/7] feat(cluster.tf): add support to set initial release
 channel version (#1625)

Co-authored-by: Andrew Peabody <andrewpeabody@google.com>
---
 autogen/main/cluster.tf.tmpl                  |  2 +-
 cluster.tf                                    |  2 +-
 .../README.md                                 | 49 ++++++++++++
 .../main.tf                                   | 76 +++++++++++++++++++
 .../outputs.tf                                | 35 +++++++++
 .../test_outputs.tf                           |  1 +
 .../variables.tf                              | 54 +++++++++++++
 .../versions.tf                               | 28 +++++++
 .../beta-autopilot-private-cluster/cluster.tf |  2 +-
 .../beta-autopilot-public-cluster/cluster.tf  |  2 +-
 .../cluster.tf                                |  2 +-
 modules/beta-private-cluster/cluster.tf       |  2 +-
 .../cluster.tf                                |  2 +-
 modules/beta-public-cluster/cluster.tf        |  2 +-
 .../private-cluster-update-variant/cluster.tf |  2 +-
 modules/private-cluster/cluster.tf            |  2 +-
 test/fixtures/shared/variables.tf             |  6 ++
 .../example.tf                                | 30 ++++++++
 .../network.tf                                | 46 +++++++++++
 .../outputs.tf                                |  1 +
 .../variables.tf                              |  1 +
 21 files changed, 337 insertions(+), 10 deletions(-)
 create mode 100644 examples/simple_regional_private_with_cluster_version/README.md
 create mode 100644 examples/simple_regional_private_with_cluster_version/main.tf
 create mode 100644 examples/simple_regional_private_with_cluster_version/outputs.tf
 create mode 120000 examples/simple_regional_private_with_cluster_version/test_outputs.tf
 create mode 100644 examples/simple_regional_private_with_cluster_version/variables.tf
 create mode 100644 examples/simple_regional_private_with_cluster_version/versions.tf
 create mode 100644 test/fixtures/simple_regional_private_with_cluster_version/example.tf
 create mode 100644 test/fixtures/simple_regional_private_with_cluster_version/network.tf
 create mode 120000 test/fixtures/simple_regional_private_with_cluster_version/outputs.tf
 create mode 120000 test/fixtures/simple_regional_private_with_cluster_version/variables.tf

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
index 1f645109b8..d263170b10 100644
--- a/autogen/main/cluster.tf.tmpl
+++ b/autogen/main/cluster.tf.tmpl
@@ -83,7 +83,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
 {% if beta_cluster and autopilot_cluster != true %}
   dynamic "cluster_telemetry" {
diff --git a/cluster.tf b/cluster.tf
index 694bc2eb05..a6f45f0a3f 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -69,7 +69,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   # only one of logging/monitoring_service or logging/monitoring_config can be specified
   logging_service = local.logmon_config_is_set ? null : var.logging_service
diff --git a/examples/simple_regional_private_with_cluster_version/README.md b/examples/simple_regional_private_with_cluster_version/README.md
new file mode 100644
index 0000000000..160fc74a4c
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/README.md
@@ -0,0 +1,49 @@
+# Simple Regional Cluster
+
+This example illustrates how to create a simple private cluster with beta features.
+
+[^]: (autogen_docs_start)
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no |
+| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
+| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
+| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes |
+| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
+| istio | Boolean to enable / disable Istio | string | `"true"` | no |
+| network | The VPC network to host the cluster in | string | n/a | yes |
+| project\_id | The project ID to host the cluster in | string | n/a | yes |
+| region | The region to host the cluster in | string | n/a | yes |
+| subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate |  |
+| client\_token |  |
+| cluster\_name | Cluster name |
+| credentials\_path |  |
+| ip\_range\_pods | The secondary IP range used for pods |
+| ip\_range\_services | The secondary IP range used for services |
+| kubernetes\_endpoint |  |
+| location |  |
+| master\_kubernetes\_version | The master Kubernetes version |
+| network |  |
+| project\_id |  |
+| region |  |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnetwork |  |
+| zones | List of zones in which the cluster resides |
+
+[^]: (autogen_docs_end)
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure
diff --git a/examples/simple_regional_private_with_cluster_version/main.tf b/examples/simple_regional_private_with_cluster_version/main.tf
new file mode 100644
index 0000000000..87b415afb5
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/main.tf
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type = "simple-regional-private"
+}
+
+data "google_client_config" "default" {}
+
+provider "kubernetes" {
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+}
+
+data "google_compute_subnetwork" "subnetwork" {
+  name    = var.subnetwork
+  project = var.project_id
+  region  = var.region
+}
+
+module "gke" {
+  source                    = "../../modules/private-cluster/"
+  project_id                = var.project_id
+  name                      = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                  = true
+  region                    = var.region
+  network                   = var.network
+  kubernetes_version        = var.kubernetes_version
+  subnetwork                = var.subnetwork
+  ip_range_pods             = var.ip_range_pods
+  ip_range_services         = var.ip_range_services
+  create_service_account    = false
+  service_account           = var.compute_engine_service_account
+  enable_private_endpoint   = true
+  enable_private_nodes      = true
+  master_ipv4_cidr_block    = "172.16.0.0/28"
+  default_max_pods_per_node = 20
+  remove_default_node_pool  = true
+
+  node_pools = [
+    {
+      name              = "pool-01"
+      min_count         = 1
+      max_count         = 100
+      local_ssd_count   = 0
+      disk_size_gb      = 100
+      disk_type         = "pd-standard"
+      auto_repair       = true
+      auto_upgrade      = true
+      service_account   = var.compute_engine_service_account
+      preemptible       = false
+      max_pods_per_node = 12
+    },
+  ]
+
+  master_authorized_networks = [
+    {
+      cidr_block   = data.google_compute_subnetwork.subnetwork.ip_cidr_range
+      display_name = "VPC"
+    },
+  ]
+}
diff --git a/examples/simple_regional_private_with_cluster_version/outputs.tf b/examples/simple_regional_private_with_cluster_version/outputs.tf
new file mode 100644
index 0000000000..01a13147c2
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/outputs.tf
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.gke.endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  value = module.gke.ca_certificate
+}
+
+output "service_account" {
+  description = "The default service account used for running nodes."
+  value       = module.gke.service_account
+}
+
diff --git a/examples/simple_regional_private_with_cluster_version/test_outputs.tf b/examples/simple_regional_private_with_cluster_version/test_outputs.tf
new file mode 120000
index 0000000000..17b34213ba
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/test_outputs.tf
@@ -0,0 +1 @@
+../../test/fixtures/all_examples/test_outputs.tf
\ No newline at end of file
diff --git a/examples/simple_regional_private_with_cluster_version/variables.tf b/examples/simple_regional_private_with_cluster_version/variables.tf
new file mode 100644
index 0000000000..5bc992dd49
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/variables.tf
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
+
+variable "cluster_name_suffix" {
+  description = "A suffix to append to the default cluster name"
+  default     = ""
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+}
+
+variable "network" {
+  description = "The VPC network to host the cluster in"
+}
+
+variable "subnetwork" {
+  description = "The subnetwork to host the cluster in"
+}
+
+variable "ip_range_pods" {
+  description = "The secondary ip range to use for pods"
+}
+
+variable "ip_range_services" {
+  description = "The secondary ip range to use for services"
+}
+
+variable "compute_engine_service_account" {
+  description = "Service account to associate to the nodes in the cluster"
+}
+
+variable "kubernetes_version" {
+  type        = string
+  description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region."
+  default     = "latest"
+}
diff --git a/examples/simple_regional_private_with_cluster_version/versions.tf b/examples/simple_regional_private_with_cluster_version/versions.tf
new file mode 100644
index 0000000000..e8fbb1aadd
--- /dev/null
+++ b/examples/simple_regional_private_with_cluster_version/versions.tf
@@ -0,0 +1,28 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.0"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 0.13"
+}
diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf
index 9d40a5aa50..a4432a6584 100644
--- a/modules/beta-autopilot-private-cluster/cluster.tf
+++ b/modules/beta-autopilot-private-cluster/cluster.tf
@@ -67,7 +67,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   cluster_autoscaling {
     dynamic "auto_provisioning_defaults" {
diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf
index e6d4920b4f..84c48a6735 100644
--- a/modules/beta-autopilot-public-cluster/cluster.tf
+++ b/modules/beta-autopilot-public-cluster/cluster.tf
@@ -67,7 +67,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   cluster_autoscaling {
     dynamic "auto_provisioning_defaults" {
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 926b3a5232..7f75a953db 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -75,7 +75,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   dynamic "cluster_telemetry" {
     for_each = local.cluster_telemetry_type_is_set ? [1] : []
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index fc2d22d5eb..63cbe46e99 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -75,7 +75,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   dynamic "cluster_telemetry" {
     for_each = local.cluster_telemetry_type_is_set ? [1] : []
diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
index 2a7b1be0b9..98738a3cf8 100644
--- a/modules/beta-public-cluster-update-variant/cluster.tf
+++ b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -75,7 +75,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   dynamic "cluster_telemetry" {
     for_each = local.cluster_telemetry_type_is_set ? [1] : []
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index e5d70fef24..754747b791 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -75,7 +75,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   dynamic "cluster_telemetry" {
     for_each = local.cluster_telemetry_type_is_set ? [1] : []
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index adfea3660a..d54761c189 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -69,7 +69,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   # only one of logging/monitoring_service or logging/monitoring_config can be specified
   logging_service = local.logmon_config_is_set ? null : var.logging_service
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index ae67d7d403..22cceeb2f0 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -69,7 +69,7 @@ resource "google_container_cluster" "primary" {
     disabled = var.disable_default_snat
   }
 
-  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
+  min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : var.kubernetes_version == "latest" ? null : var.kubernetes_version
 
   # only one of logging/monitoring_service or logging/monitoring_config can be specified
   logging_service = local.logmon_config_is_set ? null : var.logging_service
diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf
index 4465351548..8cf5823b20 100644
--- a/test/fixtures/shared/variables.tf
+++ b/test/fixtures/shared/variables.tf
@@ -39,3 +39,9 @@ variable "registry_project_ids" {
   description = "Projects to use for granting access to GCR registries, if requested"
   type        = list(string)
 }
+
+variable "kubernetes_version" {
+  type        = string
+  description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region."
+  default     = "latest"
+}
diff --git a/test/fixtures/simple_regional_private_with_cluster_version/example.tf b/test/fixtures/simple_regional_private_with_cluster_version/example.tf
new file mode 100644
index 0000000000..85c8b88b99
--- /dev/null
+++ b/test/fixtures/simple_regional_private_with_cluster_version/example.tf
@@ -0,0 +1,30 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "example" {
+  source = "../../../examples/simple_regional_private_with_cluster_version"
+
+  project_id                     = var.project_ids[1]
+  cluster_name_suffix            = "-${random_string.suffix.result}"
+  kubernetes_version             = var.kubernetes_version
+  region                         = var.region
+  network                        = google_compute_network.main.name
+  subnetwork                     = google_compute_subnetwork.main.name
+  ip_range_pods                  = google_compute_subnetwork.main.secondary_ip_range[0].range_name
+  ip_range_services              = google_compute_subnetwork.main.secondary_ip_range[1].range_name
+  compute_engine_service_account = var.compute_engine_service_accounts[1]
+}
+
diff --git a/test/fixtures/simple_regional_private_with_cluster_version/network.tf b/test/fixtures/simple_regional_private_with_cluster_version/network.tf
new file mode 100644
index 0000000000..8d643281e1
--- /dev/null
+++ b/test/fixtures/simple_regional_private_with_cluster_version/network.tf
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}
+
+resource "google_compute_network" "main" {
+  project                 = var.project_ids[1]
+  name                    = "cft-gke-test-${random_string.suffix.result}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "main" {
+  project       = var.project_ids[1]
+  name          = "cft-gke-test-${random_string.suffix.result}"
+  ip_cidr_range = "10.0.0.0/17"
+  region        = var.region
+  network       = google_compute_network.main.self_link
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-pods-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.0.0/18"
+  }
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-services-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.64.0/18"
+  }
+}
+
diff --git a/test/fixtures/simple_regional_private_with_cluster_version/outputs.tf b/test/fixtures/simple_regional_private_with_cluster_version/outputs.tf
new file mode 120000
index 0000000000..726bdc722f
--- /dev/null
+++ b/test/fixtures/simple_regional_private_with_cluster_version/outputs.tf
@@ -0,0 +1 @@
+../shared/outputs.tf
\ No newline at end of file
diff --git a/test/fixtures/simple_regional_private_with_cluster_version/variables.tf b/test/fixtures/simple_regional_private_with_cluster_version/variables.tf
new file mode 120000
index 0000000000..c113c00a3d
--- /dev/null
+++ b/test/fixtures/simple_regional_private_with_cluster_version/variables.tf
@@ -0,0 +1 @@
+../shared/variables.tf
\ No newline at end of file

From 2fe171500c3496eb88b9fdf83318b0927d225e4b Mon Sep 17 00:00:00 2001
From: Andrew Peabody <andrewpeabody@google.com>
Date: Fri, 2 Jun 2023 11:37:15 -0700
Subject: [PATCH 7/7] fix: update policy-essentials hash 59f4695 using ref
 (#1659)

---
 examples/simple_zonal_with_acm/acm.tf | 2 +-
 modules/acm/policy_bundles.tf         | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf
index e4662d08f9..61b2b61bce 100644
--- a/examples/simple_zonal_with_acm/acm.tf
+++ b/examples/simple_zonal_with_acm/acm.tf
@@ -28,7 +28,7 @@ module "acm" {
 
   secret_type = "ssh"
 
-  policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"]
+  policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022?ref=59f4695394285078f7c2029ec7d0f9ed1d6d700a"]
 
   create_metrics_gcp_sa = true
 }
diff --git a/modules/acm/policy_bundles.tf b/modules/acm/policy_bundles.tf
index 8bd122d1e0..b03987e867 100644
--- a/modules/acm/policy_bundles.tf
+++ b/modules/acm/policy_bundles.tf
@@ -18,13 +18,12 @@ module "policy_bundles" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version = "~> 3.1"
 
-  # Use index as name to avoid long url or special filesystem chars
-  for_each                = { for i, v in var.policy_bundles : i => v }
+  for_each                = toset(var.policy_bundles)
   project_id              = var.project_id
   cluster_name            = var.cluster_name
   cluster_location        = var.location
-  kubectl_create_command  = "kubectl apply -k ${each.value}"
-  kubectl_destroy_command = "kubectl delete -k ${each.value}"
+  kubectl_create_command  = "kubectl apply -k ${each.key}"
+  kubectl_destroy_command = "kubectl delete -k ${each.key}"
 
   module_depends_on = [time_sleep.wait_acm]
 }