From e3a05d800ab985f0b1b47407e1c291797ce1fc1b Mon Sep 17 00:00:00 2001
From: Andriy Kopachevskyy <kopachevsky@gmail.com>
Date: Fri, 4 Oct 2019 18:34:01 +0300
Subject: [PATCH 01/90] Add submodule to set up ACM

Added standalone ACM submodule, embedded acm to examples/simple_zonal.

Fixes #268
---
 examples/simple_zonal/README.md          |  4 ++
 examples/simple_zonal/acm.tf             | 25 +++++++
 examples/simple_zonal/outputs.tf         |  4 ++
 examples/simple_zonal/variables.tf       | 17 +++++
 modules/acm/.gitignore                   |  3 +
 modules/acm/README.md                    | 41 +++++++++++
 modules/acm/main.tf                      | 90 ++++++++++++++++++++++++
 modules/acm/outputs.tf                   |  4 ++
 modules/acm/scripts/kubectl_wrapper.sh   | 53 ++++++++++++++
 modules/acm/templates/acm-config.yml.tpl | 12 ++++
 modules/acm/variables.tf                 | 30 ++++++++
 test/setup/iam.tf                        |  1 +
 test/task_helper_functions.sh            |  2 +-
 13 files changed, 285 insertions(+), 1 deletion(-)
 create mode 100644 examples/simple_zonal/acm.tf
 create mode 100644 modules/acm/.gitignore
 create mode 100644 modules/acm/README.md
 create mode 100644 modules/acm/main.tf
 create mode 100644 modules/acm/outputs.tf
 create mode 100755 modules/acm/scripts/kubectl_wrapper.sh
 create mode 100644 modules/acm/templates/acm-config.yml.tpl
 create mode 100644 modules/acm/variables.tf

diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md
index 691f95c719..265977c33b 100644
--- a/examples/simple_zonal/README.md
+++ b/examples/simple_zonal/README.md
@@ -7,6 +7,9 @@ This example illustrates how to create a simple cluster.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| acm\_policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | `"foo-corp"` | no |
+| acm\_sync\_branch | Anthos config management Git branch | string | `"1.0.0"` | no |
+| acm\_sync\_repo | Anthos config management Git repo | string | `"git@github.com:GoogleCloudPlatform/csp-config-management.git"` | no |
 | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
@@ -20,6 +23,7 @@ This example illustrates how to create a simple cluster.
 
 | Name | Description |
 |------|-------------|
+| acm\_git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. |
 | ca\_certificate |  |
 | client\_token |  |
 | cluster\_name | Cluster name |
diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal/acm.tf
new file mode 100644
index 0000000000..1c38d22c99
--- /dev/null
+++ b/examples/simple_zonal/acm.tf
@@ -0,0 +1,25 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "acm" {
+  source       = "../../modules/acm"
+  project_id   = var.project_id
+  location     = module.gke.location
+  cluster_name = module.gke.name
+  sync_repo    = var.acm_sync_repo
+  sync_branch  = var.acm_sync_branch
+  policy_dir   = var.acm_policy_dir
+}
diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf
index 0d972dcd88..143710f8b2 100644
--- a/examples/simple_zonal/outputs.tf
+++ b/examples/simple_zonal/outputs.tf
@@ -33,3 +33,7 @@ output "service_account" {
   value       = module.gke.service_account
 }
 
+output "acm_git_creds_public" {
+  description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository."
+  value       = module.acm.git_creds_public
+}
\ No newline at end of file
diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal/variables.tf
index 516116557c..adc92dba17 100644
--- a/examples/simple_zonal/variables.tf
+++ b/examples/simple_zonal/variables.tf
@@ -48,3 +48,20 @@ variable "ip_range_services" {
   description = "The secondary ip range to use for pods"
 }
 
+variable "acm_sync_repo" {
+  description = "Anthos config management Git repo"
+  type        = string
+  default     = "git@github.com:GoogleCloudPlatform/csp-config-management.git"
+}
+
+variable "acm_sync_branch" {
+  description = "Anthos config management Git branch"
+  type        = string
+  default     = "1.0.0"
+}
+
+variable "acm_policy_dir" {
+  description = "Subfolder containing configs in Ahtons config management Git repo"
+  type        = string
+  default     = "foo-corp"
+}
\ No newline at end of file
diff --git a/modules/acm/.gitignore b/modules/acm/.gitignore
new file mode 100644
index 0000000000..aee8cb6749
--- /dev/null
+++ b/modules/acm/.gitignore
@@ -0,0 +1,3 @@
+# This fill will be always downloaded by terraform local-exec command from gc bucket
+config-management-operator.yaml
+/terraform.tfvars
diff --git a/modules/acm/README.md b/modules/acm/README.md
new file mode 100644
index 0000000000..99a40bc840
--- /dev/null
+++ b/modules/acm/README.md
@@ -0,0 +1,41 @@
+# Terraform Kubernetes Engine ACM Submodule 
+
+This module installs Anthos Config Management (ACM) in a Kubernetes cluster. 
+To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/).
+
+
+## Configure a Service Account
+
+In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md)
+plus **roles/container.admin** role.
+
+## Usage example 
+
+See [examples/simple_zonal](../../examples/simple_zonal) cluster example.
+
+## Installation 
+
+Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page.
+Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public**
+output variable to your account at your Git server.
+
+
+ <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| cluster\_name | The name of the cluster. | string | n/a | yes |
+| location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes |
+| policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes |
+| project\_id | The project in which the resource belongs. | string | n/a | yes |
+| sync\_branch | Anthos config management Git branch | string | `"master"` | no |
+| sync\_repo | Anthos config management Git repo | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. |
+
+ <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
\ No newline at end of file
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
new file mode 100644
index 0000000000..7a2a6019d4
--- /dev/null
+++ b/modules/acm/main.tf
@@ -0,0 +1,90 @@
+locals {
+  cluster_endpoint       = "https://${data.google_container_cluster.primary.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate
+}
+
+data "google_container_cluster" "primary" {
+  name     = var.cluster_name
+  project  = var.project_id
+  location = var.location
+}
+
+data "google_client_config" "default" {
+}
+
+resource "tls_private_key" "git_creds" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "null_resource" "acm_operator_config" {
+  provisioner "local-exec" {
+    command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
+  }
+
+  provisioner "local-exec" {
+    when    = "destroy"
+    command = "rm -f ${path.module}/config-management-operator.yaml"
+  }
+}
+
+resource "null_resource" "acm_operator" {
+  provisioner "local-exec" {
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${path.module}/config-management-operator.yaml"
+  }
+
+  provisioner "local-exec" {
+    when    = "destroy"
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml"
+  }
+
+  depends_on = [
+    null_resource.acm_operator_config,
+    data.google_client_config.default,
+    data.google_container_cluster.primary,
+  ]
+}
+
+resource "null_resource" "git_creds_secret" {
+  provisioner "local-exec" {
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${tls_private_key.git_creds.private_key_pem}'"
+  }
+
+  provisioner "local-exec" {
+    when    = "destroy"
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete secret git-creds -n=config-management-system"
+  }
+
+  depends_on = [
+    null_resource.acm_operator
+  ]
+}
+
+data "template_file" "acm_config" {
+  template = file("${path.module}/templates/acm-config.yml.tpl")
+
+  vars = {
+    cluster_name = var.cluster_name
+    sync_repo    = var.sync_repo
+    sync_branch  = var.sync_branch
+    policy_dir   = var.policy_dir
+    secret_type  = "ssh"
+  }
+}
+
+resource "null_resource" "acm_config" {
+  provisioner "local-exec" {
+    command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f -"
+  }
+
+  provisioner "local-exec" {
+    when    = "destroy"
+    command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f -"
+  }
+
+  depends_on = [
+    null_resource.acm_operator,
+    null_resource.git_creds_secret,
+  ]
+}
\ No newline at end of file
diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf
new file mode 100644
index 0000000000..b37d521c0e
--- /dev/null
+++ b/modules/acm/outputs.tf
@@ -0,0 +1,4 @@
+output "git_creds_public" {
+  description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository."
+  value       = tls_private_key.git_creds.public_key_openssh
+}
\ No newline at end of file
diff --git a/modules/acm/scripts/kubectl_wrapper.sh b/modules/acm/scripts/kubectl_wrapper.sh
new file mode 100755
index 0000000000..e92300bcb5
--- /dev/null
+++ b/modules/acm/scripts/kubectl_wrapper.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -e
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+HOST=$1
+TOKEN=$2
+CA_CERTIFICATE=$3
+
+shift 3
+
+RANDOM_ID="${RANDOM}_${RANDOM}"
+export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}"
+
+function cleanup {
+    rm -rf "${TMPDIR}"
+}
+trap cleanup EXIT
+
+mkdir "${TMPDIR}"
+
+export KUBECONFIG="${TMPDIR}/config"
+
+# shellcheck disable=SC1117
+base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d"
+echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate"
+
+kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null
+rm -f "${TMPDIR}/ca_certificate"
+kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null
+kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null
+kubectl config use-context kubectl-wrapper 1>/dev/null
+kubectl version 1>/dev/null
+
+"$@"
diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl
new file mode 100644
index 0000000000..af60b8690a
--- /dev/null
+++ b/modules/acm/templates/acm-config.yml.tpl
@@ -0,0 +1,12 @@
+apiVersion: configmanagement.gke.io/v1
+kind: ConfigManagement
+metadata:
+  name: config-management
+spec:
+  # clusterName is required and must be unique among all managed clusters
+  clusterName: ${cluster_name}
+  git:
+    syncRepo: ${sync_repo}
+    syncBranch: ${sync_branch}
+    secretType: ${secret_type}
+    policyDir: ${policy_dir}
\ No newline at end of file
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
new file mode 100644
index 0000000000..9640135b24
--- /dev/null
+++ b/modules/acm/variables.tf
@@ -0,0 +1,30 @@
+variable "cluster_name" {
+  description = "The name of the cluster."
+  type        = string
+}
+
+variable "project_id" {
+  description = "The project in which the resource belongs."
+  type        = string
+}
+
+variable "location" {
+  description = "The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified."
+  type        = string
+}
+
+variable "sync_repo" {
+  description = "Anthos config management Git repo"
+  type        = string
+}
+
+variable "sync_branch" {
+  description = "Anthos config management Git branch"
+  type        = string
+  default     = "master"
+}
+
+variable "policy_dir" {
+  description = "Subfolder containing configs in Ahtons config management Git repo"
+  type        = string
+}
\ No newline at end of file
diff --git a/test/setup/iam.tf b/test/setup/iam.tf
index 29facd32a9..fd8a62a18f 100644
--- a/test/setup/iam.tf
+++ b/test/setup/iam.tf
@@ -18,6 +18,7 @@ locals {
   int_required_roles = [
     "roles/cloudkms.cryptoKeyEncrypterDecrypter",
     "roles/compute.networkAdmin",
+    "roles/container.admin",
     "roles/container.clusterAdmin",
     "roles/container.developer",
     "roles/iam.serviceAccountAdmin",
diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh
index 70ab3db5c8..ddedde4b4c 100755
--- a/test/task_helper_functions.sh
+++ b/test/task_helper_functions.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env bash{
 
 # Copyright 2019 Google LLC
 #

From 9249a528c60731a627ea0682914665c236ed0a9a Mon Sep 17 00:00:00 2001
From: Kostas Evangelou <kostas.evangelou@revolut.com>
Date: Wed, 16 Oct 2019 15:54:57 +0100
Subject: [PATCH 02/90] Make default metadata, labels and tags optional in beta
 private cluster

---
 modules/beta-private-cluster/cluster.tf | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index c481c69a35..08332c4db5 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -243,22 +243,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -277,8 +269,8 @@ resource "google_container_node_pool" "pools" {
       }
     }
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )

From dcb2b11ff4206110f54d79199c6fbf125edd0966 Mon Sep 17 00:00:00 2001
From: Kostas Evangelou <kostas.evangelou@revolut.com>
Date: Wed, 16 Oct 2019 17:02:44 +0100
Subject: [PATCH 03/90] generate changes for all (sub)modules

---
 autogen/cluster.tf                            | 20 +++++------------
 cluster.tf                                    | 20 +++++------------
 .../cluster.tf                                | 22 ++++++-------------
 modules/beta-public-cluster/cluster.tf        | 20 +++++------------
 .../private-cluster-update-variant/cluster.tf | 22 ++++++-------------
 modules/private-cluster/cluster.tf            | 20 +++++------------
 6 files changed, 38 insertions(+), 86 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 296b2818df..5de8ad6876 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -341,22 +341,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -377,8 +369,8 @@ resource "google_container_node_pool" "pools" {
     }
     {% endif %}
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )
diff --git a/cluster.tf b/cluster.tf
index ffdb27b0fc..ba117abd3d 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -161,22 +161,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -184,8 +176,8 @@ resource "google_container_node_pool" "pools" {
       },
     )
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index cf1def945d..6b02e72750 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -315,22 +315,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -349,8 +341,8 @@ resource "google_container_node_pool" "pools" {
       }
     }
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )
@@ -389,7 +381,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   lifecycle {
-    ignore_changes        = [initial_node_count]
+    ignore_changes = [initial_node_count]
     create_before_destroy = true
   }
 
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index a264e932b9..0ff13c98b2 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -238,22 +238,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -272,8 +264,8 @@ resource "google_container_node_pool" "pools" {
       }
     }
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index e8db91a77a..19929ed7de 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -238,22 +238,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -261,8 +253,8 @@ resource "google_container_node_pool" "pools" {
       },
     )
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )
@@ -293,7 +285,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   lifecycle {
-    ignore_changes        = [initial_node_count]
+    ignore_changes = [initial_node_count]
     create_before_destroy = true
   }
 
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index 412e8295ed..0a9df965ef 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -166,22 +166,14 @@ resource "google_container_node_pool" "pools" {
     image_type   = lookup(var.node_pools[count.index], "image_type", "COS")
     machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")
     labels = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_labels["all"],
       var.node_pools_labels[var.node_pools[count.index]["name"]],
     )
     metadata = merge(
-      {
-        "cluster_name" = var.name
-      },
-      {
-        "node_pool" = var.node_pools[count.index]["name"]
-      },
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
+      lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {},
       var.node_pools_metadata["all"],
       var.node_pools_metadata[var.node_pools[count.index]["name"]],
       {
@@ -189,8 +181,8 @@ resource "google_container_node_pool" "pools" {
       },
     )
     tags = concat(
-      ["gke-${var.name}"],
-      ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],
+      lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
+      lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [],
       var.node_pools_tags["all"],
       var.node_pools_tags[var.node_pools[count.index]["name"]],
     )

From 970671f7d42705147a07e78375dced78332bdaab Mon Sep 17 00:00:00 2001
From: Kostas Evangelou <kostas.evangelou@revolut.com>
Date: Wed, 16 Oct 2019 17:33:00 +0100
Subject: [PATCH 04/90] format tf files

---
 modules/beta-private-cluster-update-variant/cluster.tf | 2 +-
 modules/private-cluster-update-variant/cluster.tf      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 6b02e72750..5b3b769c9e 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -381,7 +381,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   lifecycle {
-    ignore_changes = [initial_node_count]
+    ignore_changes        = [initial_node_count]
     create_before_destroy = true
   }
 
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index 19929ed7de..c6d76de2e5 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -285,7 +285,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   lifecycle {
-    ignore_changes = [initial_node_count]
+    ignore_changes        = [initial_node_count]
     create_before_destroy = true
   }
 

From 50521cfdd65985274fda85a6188b8d89a5754cea Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 17 Oct 2019 21:12:33 -0500
Subject: [PATCH 05/90] added simple example with networking

---
 .../simple_regional_with_networking/README.md | 47 +++++++++++++
 .../simple_regional_with_networking/main.tf   | 68 +++++++++++++++++++
 .../outputs.tf                                | 48 +++++++++++++
 .../variables.tf                              | 49 +++++++++++++
 4 files changed, 212 insertions(+)
 create mode 100644 examples/simple_regional_with_networking/README.md
 create mode 100644 examples/simple_regional_with_networking/main.tf
 create mode 100644 examples/simple_regional_with_networking/outputs.tf
 create mode 100644 examples/simple_regional_with_networking/variables.tf

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
new file mode 100644
index 0000000000..a58378f9ae
--- /dev/null
+++ b/examples/simple_regional_with_networking/README.md
@@ -0,0 +1,47 @@
+# Simple Regional Cluster with Networking
+
+This example illustrates how to create a VPC and a simple cluster.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
+| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
+| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
+| network\_name | The VPC network created to host the cluster in | string | n/a | yes |
+| project\_id | The project ID to host the cluster in | string | n/a | yes |
+| region | The region to host the cluster in | string | n/a | yes |
+| subnetwork | The subnetwork created to host the cluster in | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate |  |
+| client\_token |  |
+| cluster\_name | Cluster name |
+| ip\_range\_pods | The secondary IP range used for pods |
+| ip\_range\_services | The secondary IP range used for services |
+| kubernetes\_endpoint |  |
+| location |  |
+| master\_kubernetes\_version | The master Kubernetes version |
+| network |  |
+| network\_name | The name of the VPC being created |
+| project\_id |  |
+| region |  |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnet\_names | The name of the subnet being created |
+| subnet\_secondary\_ranges | The secondary ranges associated with the subnet |
+| subnetwork |  |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
new file mode 100644
index 0000000000..f780395645
--- /dev/null
+++ b/examples/simple_regional_with_networking/main.tf
@@ -0,0 +1,68 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type = "simple-regional"
+}
+
+provider "google" {
+  version = "~> 2.12.0"
+  region  = var.region
+}
+
+module "gcp-network" {
+  source       = "terraform-google-modules/network/google"
+  project_id   = var.project_id
+  network_name = var.network_name
+
+  subnets = [
+    {
+      subnet_name   = var.subnetwork
+      subnet_ip     = "10.0.0.0/17"
+      subnet_region = var.region
+    },
+  ]
+
+  secondary_ranges = {
+    "${var.subnetwork}" = [
+      {
+        range_name    = var.ip_range_pods
+        ip_cidr_range = "192.168.0.0/18"
+      },
+      {
+        range_name    = var.ip_range_services
+        ip_cidr_range = "192.168.64.0/18"
+      },
+    ]
+  }
+}
+
+module "gke" {
+  source                 = "terraform-google-modules/kubernetes-engine/google"
+  project_id             = var.project_id
+  name                   = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional               = true
+  region                 = var.region
+  network                = var.network_name
+  subnetwork             = var.subnetwork
+  ip_range_pods          = var.ip_range_pods
+  ip_range_services      = var.ip_range_services
+  create_service_account = false
+  service_account        = var.compute_engine_service_account
+}
+
+data "google_client_config" "default" {
+}
diff --git a/examples/simple_regional_with_networking/outputs.tf b/examples/simple_regional_with_networking/outputs.tf
new file mode 100644
index 0000000000..9abceb572a
--- /dev/null
+++ b/examples/simple_regional_with_networking/outputs.tf
@@ -0,0 +1,48 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.gke.endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  value = module.gke.ca_certificate
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.gke.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = module.gcp-network.network_name
+}
+output "subnet_names" {
+  description = "The name of the subnet being created"
+  value       = module.gcp-network.subnets_names
+}
+output "subnet_secondary_ranges" {
+  description = "The secondary ranges associated with the subnet"
+  value       = flatten(module.gcp-network.subnets_secondary_ranges)
+}
+
diff --git a/examples/simple_regional_with_networking/variables.tf b/examples/simple_regional_with_networking/variables.tf
new file mode 100644
index 0000000000..d78a9fbdb7
--- /dev/null
+++ b/examples/simple_regional_with_networking/variables.tf
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
+
+variable "cluster_name_suffix" {
+  description = "A suffix to append to the default cluster name"
+  default     = ""
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+}
+
+variable "network_name" {
+  description = "The VPC network created to host the cluster in"
+}
+
+variable "subnetwork" {
+  description = "The subnetwork created to host the cluster in"
+}
+
+variable "ip_range_pods" {
+  description = "The secondary ip range to use for pods"
+}
+
+variable "ip_range_services" {
+  description = "The secondary ip range to use for pods"
+}
+
+variable "compute_engine_service_account" {
+  description = "Service account to associate to the nodes in the cluster"
+}
+

From 1668b646b7993745cac49b62997cccec70a203c0 Mon Sep 17 00:00:00 2001
From: omazin <osmansimf@rambler.ru>
Date: Fri, 18 Oct 2019 08:29:57 +0300
Subject: [PATCH 06/90] [wait-for-cluster.sh] Update the script in all
 submodules. Follow up #284.

---
 .../scripts/wait-for-cluster.sh                             | 6 +++++-
 modules/beta-private-cluster/scripts/wait-for-cluster.sh    | 6 +++++-
 modules/beta-public-cluster/scripts/wait-for-cluster.sh     | 6 +++++-
 .../scripts/wait-for-cluster.sh                             | 6 +++++-
 modules/private-cluster/scripts/wait-for-cluster.sh         | 6 +++++-
 5 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"
diff --git a/modules/beta-private-cluster/scripts/wait-for-cluster.sh b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/modules/beta-private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"
diff --git a/modules/beta-public-cluster/scripts/wait-for-cluster.sh b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/modules/beta-public-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"
diff --git a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"
diff --git a/modules/private-cluster/scripts/wait-for-cluster.sh b/modules/private-cluster/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/modules/private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

From af8f1b858ed1e7e21cf3fe32efef219cb6ab81ab Mon Sep 17 00:00:00 2001
From: omazin <osmansimf@rambler.ru>
Date: Fri, 18 Oct 2019 12:50:52 +0300
Subject: [PATCH 07/90] [wait-for-cluster.sh] Update the script in autogen
 folder.

---
 autogen/scripts/wait-for-cluster.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/autogen/scripts/wait-for-cluster.sh b/autogen/scripts/wait-for-cluster.sh
index 6ff3253d58..37f0176ec7 100755
--- a/autogen/scripts/wait-for-cluster.sh
+++ b/autogen/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,10 @@
 
 set -e
 
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

From 9c6627362467fa4dffd5a29fbafca885f4300d53 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Tue, 8 Oct 2019 20:03:03 +0300
Subject: [PATCH 08/90] Added variable `skip_provisioners` to skip 'local-exec'

 * Fix #258
 * Added test `simple_regional_skip_local_exec`
 * Remove old upgrading guide from README's
---
 README.md                                 | 17 +++++++++++++++++
 autogen/cluster.tf                        |  1 +
 autogen/dns.tf                            |  2 +-
 autogen/variables.tf                      |  5 +++++
 cluster.tf                                |  1 +
 dns.tf                                    |  2 +-
 examples/simple_regional/README.md        |  1 +
 examples/simple_regional/main.tf          |  1 +
 examples/simple_regional/variables.tf     |  5 +++++
 modules/beta-private-cluster/README.md    |  1 +
 modules/beta-private-cluster/cluster.tf   |  1 +
 modules/beta-private-cluster/dns.tf       |  2 +-
 modules/beta-private-cluster/variables.tf |  5 +++++
 modules/beta-public-cluster/README.md     |  1 +
 modules/beta-public-cluster/cluster.tf    |  1 +
 modules/beta-public-cluster/dns.tf        |  2 +-
 modules/beta-public-cluster/variables.tf  |  5 +++++
 modules/private-cluster/README.md         |  1 +
 modules/private-cluster/cluster.tf        |  1 +
 modules/private-cluster/dns.tf            |  2 +-
 modules/private-cluster/variables.tf      |  5 +++++
 test/fixtures/simple_regional/example.tf  |  1 +
 variables.tf                              |  5 +++++
 23 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 50e2afd63b..ac4ff4174d 100644
--- a/README.md
+++ b/README.md
@@ -108,6 +108,22 @@ Then perform the following commands on the root folder:
 - `terraform apply` to apply the infrastructure build
 - `terraform destroy` to destroy the built infrastructure
 
+## Upgrade to v3.0.0
+
+v3.0.0 is a breaking release. Refer to the
+[Upgrading to v3.0 guide][upgrading-to-v3.0] for details.
+
+## Upgrade to v2.0.0
+
+v2.0.0 is a breaking release. Refer to the
+[Upgrading to v2.0 guide][upgrading-to-v2.0] for details.
+
+## Upgrade to v1.0.0
+
+Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths.
+
+In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
@@ -153,6 +169,7 @@ Then perform the following commands on the root folder:
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 4e5fd74d55..95ca6a31b6 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -352,6 +352,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/autogen/dns.tf b/autogen/dns.tf
index d9d4a35395..731e010b0d 100644
--- a/autogen/dns.tf
+++ b/autogen/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/autogen/variables.tf b/autogen/variables.tf
index 0fedacb2af..b3137c6dcb 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -304,6 +304,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 {% if private_cluster %}
 
 variable "deploy_using_private_endpoint" {
diff --git a/cluster.tf b/cluster.tf
index ffdb27b0fc..40b5559703 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -227,6 +227,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/dns.tf b/dns.tf
index b240a23e65..f490c15504 100644
--- a/dns.tf
+++ b/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md
index fb209e47b5..1f0a187b6b 100644
--- a/examples/simple_regional/README.md
+++ b/examples/simple_regional/README.md
@@ -14,6 +14,7 @@ This example illustrates how to create a simple cluster.
 | network | The VPC network to host the cluster in | string | n/a | yes |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
+| skip\_provisioners | Flag to skip local-exec provisioners | bool | `"false"` | no |
 | subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
 
 ## Outputs
diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf
index 4662435fbd..353ae91906 100644
--- a/examples/simple_regional/main.tf
+++ b/examples/simple_regional/main.tf
@@ -35,6 +35,7 @@ module "gke" {
   ip_range_services      = var.ip_range_services
   create_service_account = false
   service_account        = var.compute_engine_service_account
+  skip_provisioners      = var.skip_provisioners
 }
 
 data "google_client_config" "default" {
diff --git a/examples/simple_regional/variables.tf b/examples/simple_regional/variables.tf
index 6121eab9ea..e7405d9e21 100644
--- a/examples/simple_regional/variables.tf
+++ b/examples/simple_regional/variables.tf
@@ -47,3 +47,8 @@ variable "compute_engine_service_account" {
   description = "Service account to associate to the nodes in the cluster"
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip local-exec provisioners"
+  default     = false
+}
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 988d48ead8..242b46a114 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -194,6 +194,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index c481c69a35..bb19940d1f 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -328,6 +328,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf
index b240a23e65..f490c15504 100644
--- a/modules/beta-private-cluster/dns.tf
+++ b/modules/beta-private-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index 9a869a830f..b6cd988fa8 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -302,6 +302,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 
 variable "deploy_using_private_endpoint" {
   type        = bool
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 7d59e927bf..1bd0d313d2 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -185,6 +185,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index a264e932b9..e2e46ac862 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -323,6 +323,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf
index b240a23e65..f490c15504 100644
--- a/modules/beta-public-cluster/dns.tf
+++ b/modules/beta-public-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 0ae2b75661..fda9068e96 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -302,6 +302,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 
 variable "istio" {
   description = "(Beta) Enable Istio addon"
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index d823f640fa..f2d8f895ff 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -178,6 +178,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index 412e8295ed..ab8b275ab1 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -232,6 +232,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf
index b240a23e65..f490c15504 100644
--- a/modules/private-cluster/dns.tf
+++ b/modules/private-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
index 8008e08975..44544f57b4 100644
--- a/modules/private-cluster/variables.tf
+++ b/modules/private-cluster/variables.tf
@@ -292,6 +292,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 
 variable "deploy_using_private_endpoint" {
   type        = bool
diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf
index a03fadb28b..7f8bb83637 100644
--- a/test/fixtures/simple_regional/example.tf
+++ b/test/fixtures/simple_regional/example.tf
@@ -25,4 +25,5 @@ module "example" {
   ip_range_pods                  = google_compute_subnetwork.main.secondary_ip_range[0].range_name
   ip_range_services              = google_compute_subnetwork.main.secondary_ip_range[1].range_name
   compute_engine_service_account = var.compute_engine_service_account
+  skip_provisioners              = true
 }
diff --git a/variables.tf b/variables.tf
index 460bdeaeff..c7a133a1e1 100644
--- a/variables.tf
+++ b/variables.tf
@@ -292,3 +292,8 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}

From a0f5f7b9b48bc9161a18b148c0d59b5e441f23b2 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Fri, 18 Oct 2019 14:42:34 +0300
Subject: [PATCH 09/90] Moved sanbox_config to node_pool resource (Fix #240)

---
 autogen/cluster.tf                               | 16 ++++++++--------
 .../cluster.tf                                   | 16 ++++++++--------
 modules/beta-private-cluster/cluster.tf          | 16 ++++++++--------
 modules/beta-public-cluster/cluster.tf           | 16 ++++++++--------
 4 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 296b2818df..035eeb0cf7 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -167,14 +167,6 @@ resource "google_container_cluster" "primary" {
           node_metadata = workload_metadata_config.value.node_metadata
         }
       }
-
-      dynamic "sandbox_config" {
-        for_each = local.cluster_sandbox_enabled
-
-        content {
-          sandbox_type = sandbox_config.value
-        }
-      }
       {% endif %}
     }
   }
@@ -415,6 +407,14 @@ resource "google_container_node_pool" "pools" {
         node_metadata = workload_metadata_config.value.node_metadata
       }
     }
+
+    dynamic "sandbox_config" {
+      for_each = local.cluster_sandbox_enabled
+
+      content {
+        sandbox_type = sandbox_config.value
+      }
+    }
     {% endif %}
   }
 
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index cf1def945d..2348150d19 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" {
           node_metadata = workload_metadata_config.value.node_metadata
         }
       }
-
-      dynamic "sandbox_config" {
-        for_each = local.cluster_sandbox_enabled
-
-        content {
-          sandbox_type = sandbox_config.value
-        }
-      }
     }
   }
 
@@ -386,6 +378,14 @@ resource "google_container_node_pool" "pools" {
         node_metadata = workload_metadata_config.value.node_metadata
       }
     }
+
+    dynamic "sandbox_config" {
+      for_each = local.cluster_sandbox_enabled
+
+      content {
+        sandbox_type = sandbox_config.value
+      }
+    }
   }
 
   lifecycle {
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index c481c69a35..56f40ed17a 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" {
           node_metadata = workload_metadata_config.value.node_metadata
         }
       }
-
-      dynamic "sandbox_config" {
-        for_each = local.cluster_sandbox_enabled
-
-        content {
-          sandbox_type = sandbox_config.value
-        }
-      }
     }
   }
 
@@ -314,6 +306,14 @@ resource "google_container_node_pool" "pools" {
         node_metadata = workload_metadata_config.value.node_metadata
       }
     }
+
+    dynamic "sandbox_config" {
+      for_each = local.cluster_sandbox_enabled
+
+      content {
+        sandbox_type = sandbox_config.value
+      }
+    }
   }
 
   lifecycle {
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index a264e932b9..e37b2b3b31 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" {
           node_metadata = workload_metadata_config.value.node_metadata
         }
       }
-
-      dynamic "sandbox_config" {
-        for_each = local.cluster_sandbox_enabled
-
-        content {
-          sandbox_type = sandbox_config.value
-        }
-      }
     }
   }
 
@@ -309,6 +301,14 @@ resource "google_container_node_pool" "pools" {
         node_metadata = workload_metadata_config.value.node_metadata
       }
     }
+
+    dynamic "sandbox_config" {
+      for_each = local.cluster_sandbox_enabled
+
+      content {
+        sandbox_type = sandbox_config.value
+      }
+    }
   }
 
   lifecycle {

From 6a214aad3939e3904e5ca504deb414f8cb93dd19 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Fri, 18 Oct 2019 11:21:58 -0500
Subject: [PATCH 10/90] fix docs

---
 examples/simple_regional_with_networking/README.md | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index a58378f9ae..376602c232 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -22,21 +22,11 @@ This example illustrates how to create a VPC and a simple cluster.
 |------|-------------|
 | ca\_certificate |  |
 | client\_token |  |
-| cluster\_name | Cluster name |
-| ip\_range\_pods | The secondary IP range used for pods |
-| ip\_range\_services | The secondary IP range used for services |
 | kubernetes\_endpoint |  |
-| location |  |
-| master\_kubernetes\_version | The master Kubernetes version |
-| network |  |
 | network\_name | The name of the VPC being created |
-| project\_id |  |
-| region |  |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | subnet\_names | The name of the subnet being created |
 | subnet\_secondary\_ranges | The secondary ranges associated with the subnet |
-| subnetwork |  |
-| zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

From 312b65ad49189d06f13aa84de4c4423353aac21b Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Sat, 19 Oct 2019 17:55:15 -0500
Subject: [PATCH 11/90] added tests

---
 .../simple_regional_with_networking/main.tf   |   9 +-
 .../outputs.tf                                |   5 +-
 .../test_outputs.tf                           |   1 +
 .../variables.tf                              |   2 +-
 .../example.tf                                |  24 +++
 .../outputs.tf                                |  64 ++++++
 .../variables.tf                              |  35 +++
 .../controls/gcloud.rb                        | 202 ++++++++++++++++++
 .../inspec.yml                                |  30 +++
 9 files changed, 365 insertions(+), 7 deletions(-)
 create mode 120000 examples/simple_regional_with_networking/test_outputs.tf
 create mode 100644 test/fixtures/simple_regional_with_networking/example.tf
 create mode 100644 test/fixtures/simple_regional_with_networking/outputs.tf
 create mode 100644 test/fixtures/simple_regional_with_networking/variables.tf
 create mode 100644 test/integration/simple_regional_with_networking/controls/gcloud.rb
 create mode 100644 test/integration/simple_regional_with_networking/inspec.yml

diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index f780395645..1f8186e853 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -25,8 +25,9 @@ provider "google" {
 
 module "gcp-network" {
   source       = "terraform-google-modules/network/google"
+  version = "~> 1.4.0"
   project_id   = var.project_id
-  network_name = var.network_name
+  network_name = var.network
 
   subnets = [
     {
@@ -51,13 +52,13 @@ module "gcp-network" {
 }
 
 module "gke" {
-  source                 = "terraform-google-modules/kubernetes-engine/google"
+  source                 = "../../"
   project_id             = var.project_id
   name                   = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
   regional               = true
   region                 = var.region
-  network                = var.network_name
-  subnetwork             = var.subnetwork
+  network                = module.gcp-network.network_name
+  subnetwork             = module.gcp-network.subnets_names[0]
   ip_range_pods          = var.ip_range_pods
   ip_range_services      = var.ip_range_services
   create_service_account = false
diff --git a/examples/simple_regional_with_networking/outputs.tf b/examples/simple_regional_with_networking/outputs.tf
index 9abceb572a..6cf2ab5eab 100644
--- a/examples/simple_regional_with_networking/outputs.tf
+++ b/examples/simple_regional_with_networking/outputs.tf
@@ -37,12 +37,13 @@ output "network_name" {
   description = "The name of the VPC being created"
   value       = module.gcp-network.network_name
 }
+
 output "subnet_names" {
   description = "The name of the subnet being created"
   value       = module.gcp-network.subnets_names
 }
+
 output "subnet_secondary_ranges" {
   description = "The secondary ranges associated with the subnet"
-  value       = flatten(module.gcp-network.subnets_secondary_ranges)
+  value       = module.gcp-network.subnets_secondary_ranges
 }
-
diff --git a/examples/simple_regional_with_networking/test_outputs.tf b/examples/simple_regional_with_networking/test_outputs.tf
new file mode 120000
index 0000000000..17b34213ba
--- /dev/null
+++ b/examples/simple_regional_with_networking/test_outputs.tf
@@ -0,0 +1 @@
+../../test/fixtures/all_examples/test_outputs.tf
\ No newline at end of file
diff --git a/examples/simple_regional_with_networking/variables.tf b/examples/simple_regional_with_networking/variables.tf
index d78a9fbdb7..5c36a81c16 100644
--- a/examples/simple_regional_with_networking/variables.tf
+++ b/examples/simple_regional_with_networking/variables.tf
@@ -27,7 +27,7 @@ variable "region" {
   description = "The region to host the cluster in"
 }
 
-variable "network_name" {
+variable "network" {
   description = "The VPC network created to host the cluster in"
 }
 
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
new file mode 100644
index 0000000000..4322474b35
--- /dev/null
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -0,0 +1,24 @@
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}
+locals {
+  network = "gke-network-${random_string.suffix.result}"
+  subnetwork = "gke-subnetwork-${random_string.suffix.result}"
+  ip_range_pods="gke-ip-range-pods-${random_string.suffix.result}"
+  ip_range_services="gke-ip-range-svc-${random_string.suffix.result}"
+}
+module "example" {
+  source = "../../../examples/simple_regional_with_networking"
+
+  project_id                     = var.project_id
+  cluster_name_suffix            = "-${random_string.suffix.result}"
+  region                         = var.region
+  network                        = local.network
+  subnetwork                     = local.subnetwork
+  ip_range_pods                  = local.ip_range_pods
+  ip_range_services              = local.ip_range_services
+  compute_engine_service_account = var.compute_engine_service_account
+}
\ No newline at end of file
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
new file mode 100644
index 0000000000..32955a3206
--- /dev/null
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -0,0 +1,64 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+output "project_id" {
+  value = var.project_id
+}
+
+output "location" {
+  value = module.example.location
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.example.cluster_name
+}
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.example.kubernetes_endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = module.example.client_token
+}
+
+output "ca_certificate" {
+  value = module.example.ca_certificate
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.example.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = local.network
+}
+
+output "subnet_name" {
+  description = "The name of the subnet being created"
+  value       = local.subnetwork
+}
+
+output "region" {
+  description = "The region the cluster is hosted in"
+  value       = module.example.region
+}
+
diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf
new file mode 100644
index 0000000000..bc35e84fee
--- /dev/null
+++ b/test/fixtures/simple_regional_with_networking/variables.tf
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
+
+variable "cluster_name_suffix" {
+  description = "A suffix to append to the default cluster name"
+  default     = ""
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+   default     = "us-east4"
+}
+
+
+variable "compute_engine_service_account" {
+  description = "Service account to associate to the nodes in the cluster"
+}
+
diff --git a/test/integration/simple_regional_with_networking/controls/gcloud.rb b/test/integration/simple_regional_with_networking/controls/gcloud.rb
new file mode 100644
index 0000000000..939780ff18
--- /dev/null
+++ b/test/integration/simple_regional_with_networking/controls/gcloud.rb
@@ -0,0 +1,202 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project_id = attribute('project_id')
+location = attribute('location')
+cluster_name = attribute('cluster_name')
+network_name = attribute('network_name')
+subnet_name = attribute('subnet_name')
+region = attribute('region')
+
+control "gcloud" do
+  title "Google Compute Engine GKE configuration"
+  describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let!(:data) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+
+    describe "cluster" do
+      it "is running" do
+        expect(data['status']).to eq 'RUNNING'
+      end
+
+      it "is regional" do
+        expect(data['location']).to match(/^.*[1-9]$/)
+      end
+
+      it "uses public nodes and master endpoint" do
+        expect(data['privateClusterConfig']).to eq nil
+      end
+
+      it "has the expected addon settings" do
+        expect(data['addonsConfig']).to eq({
+          "horizontalPodAutoscaling" => {},
+          "httpLoadBalancing" => {},
+          "kubernetesDashboard" => {
+            "disabled" => true,
+          },
+          "networkPolicyConfig" => {
+            "disabled" => true,
+          },
+        })
+      end
+    end
+
+    describe "default node pool" do
+      let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first }
+
+      it "exists" do
+        expect(data['nodePools']).to include(
+          including(
+            "name" => "default-pool",
+          )
+        )
+      end
+    end
+
+    describe "node pool" do
+      let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } }
+
+      it "has autoscaling enabled" do
+        expect(node_pools).to include(
+          including(
+            "autoscaling" => including(
+              "enabled" => true,
+            ),
+          )
+        )
+      end
+
+      it "has the expected minimum node count" do
+        expect(node_pools).to include(
+          including(
+            "autoscaling" => including(
+              "minNodeCount" => 1,
+            ),
+          )
+        )
+      end
+
+      it "has the expected maximum node count" do
+        expect(node_pools).to include(
+          including(
+            "autoscaling" => including(
+              "maxNodeCount" => 100,
+            ),
+          )
+        )
+      end
+
+      it "is the expected machine type" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "machineType" => "n1-standard-2",
+            ),
+          )
+        )
+      end
+
+      it "has the expected disk size" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "diskSizeGb" => 100,
+            ),
+          )
+        )
+      end
+
+      it "has the expected labels" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "labels" => including(
+                "cluster_name" => cluster_name,
+                "node_pool" => "default-node-pool",
+              ),
+            ),
+          )
+        )
+      end
+
+      it "has the expected network tags" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "tags" => match_array([
+                "gke-#{cluster_name}",
+                "gke-#{cluster_name}-default-node-pool",
+              ]),
+            ),
+          )
+        )
+      end
+
+      it "has autorepair enabled" do
+        expect(node_pools).to include(
+          including(
+            "management" => including(
+              "autoRepair" => true,
+            ),
+          )
+        )
+      end
+
+      it "has autoupgrade enabled" do
+        expect(node_pools).to include(
+          including(
+            "management" => including(
+              "autoUpgrade" => true,
+            ),
+          )
+        )
+      end
+    end
+  end
+end
+
+control "network" do
+  title "gcp network configuration"
+
+  describe google_compute_network(
+    project: project_id,
+    name: network_name
+  ) do
+    it { should exist }
+  end
+end
+control "subnetwork" do
+  title "gcp subnetwork configuration"
+
+  describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let(:data) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/inspec.yml b/test/integration/simple_regional_with_networking/inspec.yml
new file mode 100644
index 0000000000..9f70b7327c
--- /dev/null
+++ b/test/integration/simple_regional_with_networking/inspec.yml
@@ -0,0 +1,30 @@
+name: simple_regional_with_networking
+depends:
+  - name: inspec-gcp
+    git: https://github.com/inspec/inspec-gcp.git
+    tag: v0.11.0
+attributes:
+  - name: project_id
+    required: true
+    type: string
+  - name: location
+    required: true
+    type: string
+  - name: cluster_name
+    required: true
+    type: string
+  - name: kubernetes_endpoint
+    required: true
+    type: string
+  - name: client_token
+    required: true
+    type: string
+  - name: network_name
+    required: true
+    type: string
+  - name: subnet_name
+    required: true
+    type: string
+  - name: region
+    required: true
+    type: string

From 65d9e25b17e47c2b883e81bd8c3e23adac455f50 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Sat, 19 Oct 2019 18:25:51 -0500
Subject: [PATCH 12/90] add test suite

---
 .kitchen.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.kitchen.yml b/.kitchen.yml
index 9f5df5a03e..89dc11d0d3 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -68,6 +68,13 @@ suites:
       systems:
         - name: simple_regional
           backend: local
+  - name: "simple_regional_with_networking"
+    driver:
+      root_module_directory: test/fixtures/simple_regional_with_networking
+    verifier:
+      systems:
+        - name: simple_regional_with_networking
+          backend: local
   - name: "simple_regional_private"
     driver:
       root_module_directory: test/fixtures/simple_regional_private

From 9413187aad59b113b6404f5ee307eb0c548315fb Mon Sep 17 00:00:00 2001
From: Bharath Baiju <bbaiju@bbaiju-macbookpro.roam.corp.google.com>
Date: Sat, 19 Oct 2019 19:36:50 -0500
Subject: [PATCH 13/90] fix inspec gcp tests

---
 .kitchen.yml                                  | 10 ++++
 .../example.tf                                | 14 ++++++
 .../outputs.tf                                |  9 ++++
 .../controls/gcloud.rb                        | 30 ------------
 .../controls/network.rb                       | 32 ++++++++++++
 .../controls/subnet.rb                        | 49 +++++++++++++++++++
 .../inspec.yml                                |  8 ++-
 7 files changed, 121 insertions(+), 31 deletions(-)
 create mode 100644 test/integration/simple_regional_with_networking/controls/network.rb
 create mode 100644 test/integration/simple_regional_with_networking/controls/subnet.rb

diff --git a/.kitchen.yml b/.kitchen.yml
index 89dc11d0d3..7b08045314 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -75,6 +75,16 @@ suites:
       systems:
         - name: simple_regional_with_networking
           backend: local
+          controls:
+            - gcloud
+        - name: subnet
+          backend: local
+          controls:
+            - subnet
+        - name: network
+          backend: gcp
+          controls:
+            - network
   - name: "simple_regional_private"
     driver:
       root_module_directory: test/fixtures/simple_regional_private
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index 4322474b35..d0d6b3b91d 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -1,4 +1,18 @@
 
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 resource "random_string" "suffix" {
   length  = 4
   special = false
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
index 32955a3206..31da62d519 100644
--- a/test/fixtures/simple_regional_with_networking/outputs.tf
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -62,3 +62,12 @@ output "region" {
   value       = module.example.region
 }
 
+output "ip_range_pods_name" {
+  description = "The secondary range name for pods"
+  value       = local.ip_range_pods
+}
+
+output "ip_range_services_name" {
+  description = "The secondary range name for services"
+  value       = local.ip_range_services
+}
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/controls/gcloud.rb b/test/integration/simple_regional_with_networking/controls/gcloud.rb
index 939780ff18..e6152ec2a0 100644
--- a/test/integration/simple_regional_with_networking/controls/gcloud.rb
+++ b/test/integration/simple_regional_with_networking/controls/gcloud.rb
@@ -15,9 +15,6 @@
 project_id = attribute('project_id')
 location = attribute('location')
 cluster_name = attribute('cluster_name')
-network_name = attribute('network_name')
-subnet_name = attribute('subnet_name')
-region = attribute('region')
 
 control "gcloud" do
   title "Google Compute Engine GKE configuration"
@@ -172,31 +169,4 @@
       end
     end
   end
-end
-
-control "network" do
-  title "gcp network configuration"
-
-  describe google_compute_network(
-    project: project_id,
-    name: network_name
-  ) do
-    it { should exist }
-  end
-end
-control "subnetwork" do
-  title "gcp subnetwork configuration"
-
-  describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do
-    its(:exit_status) { should eq 0 }
-    its(:stderr) { should eq '' }
-
-    let(:data) do
-      if subject.exit_status == 0
-        JSON.parse(subject.stdout)
-      else
-        {}
-      end
-    end
-  end
 end
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/controls/network.rb b/test/integration/simple_regional_with_networking/controls/network.rb
new file mode 100644
index 0000000000..db356efe28
--- /dev/null
+++ b/test/integration/simple_regional_with_networking/controls/network.rb
@@ -0,0 +1,32 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project_id = attribute('project_id')
+network_name = attribute('network_name')
+subnet_name = attribute('subnet_name')
+
+control "network" do
+    title "gcp network configuration"
+    
+    describe google_compute_network(
+      project: project_id,
+      name: network_name
+    ) do
+      it { should exist }
+      its ('subnetworks.count') { should eq 1 }
+      its ('subnetworks.first') { should match subnet_name}
+    end
+  end
+
+  
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/controls/subnet.rb b/test/integration/simple_regional_with_networking/controls/subnet.rb
new file mode 100644
index 0000000000..f1800e3654
--- /dev/null
+++ b/test/integration/simple_regional_with_networking/controls/subnet.rb
@@ -0,0 +1,49 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project_id = attribute('project_id')
+network_name = attribute('network_name')
+subnet_name = attribute('subnet_name')
+region = attribute('region')
+ip_range_pods_name = attribute('ip_range_pods_name')
+ip_range_services_name = attribute('ip_range_services_name')
+
+control "subnet" do
+    title "gcp subnetwork configuration"
+  
+    describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do
+      its(:exit_status) { should eq 0 }
+      its(:stderr) { should eq '' }
+  
+      let(:data) do
+        if subject.exit_status == 0
+          JSON.parse(subject.stdout)
+        else
+          {}
+        end
+      end
+      it "#should have the correct secondaryIpRanges configuration for #{ip_range_pods_name}" do
+        expect(data["secondaryIpRanges"][0]).to include(
+          "rangeName"   => ip_range_pods_name,
+          "ipCidrRange" => "192.168.0.0/18"
+        )
+      end
+      it "#should have the correct secondaryIpRanges configuration for #{ip_range_services_name}" do
+        expect(data["secondaryIpRanges"][1]).to include(
+          "rangeName"   => ip_range_services_name,
+          "ipCidrRange" => "192.168.64.0/18"
+        )
+      end
+    end
+  end
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/inspec.yml b/test/integration/simple_regional_with_networking/inspec.yml
index 9f70b7327c..bf2e4e86aa 100644
--- a/test/integration/simple_regional_with_networking/inspec.yml
+++ b/test/integration/simple_regional_with_networking/inspec.yml
@@ -2,7 +2,7 @@ name: simple_regional_with_networking
 depends:
   - name: inspec-gcp
     git: https://github.com/inspec/inspec-gcp.git
-    tag: v0.11.0
+    tag: v0.10.0
 attributes:
   - name: project_id
     required: true
@@ -28,3 +28,9 @@ attributes:
   - name: region
     required: true
     type: string
+  - name: ip_range_pods_name
+    required: true
+    type: string
+  - name: ip_range_services_name
+    required: true
+    type: string

From c4f138df9cb571355742480716b2a022a329fd3b Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Sat, 19 Oct 2019 20:14:51 -0500
Subject: [PATCH 14/90] fix lint

---
 .../simple_regional_with_networking/README.md | 12 +++++-
 .../simple_regional_with_networking/main.tf   |  2 +-
 .../example.tf                                | 39 ++++++++++---------
 .../outputs.tf                                |  2 +-
 .../variables.tf                              |  2 +-
 .../controls/gcloud.rb                        |  2 +-
 .../controls/network.rb                       |  6 +--
 .../controls/subnet.rb                        |  5 +--
 8 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index 376602c232..f39675063d 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -11,7 +11,7 @@ This example illustrates how to create a VPC and a simple cluster.
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
-| network\_name | The VPC network created to host the cluster in | string | n/a | yes |
+| network | The VPC network created to host the cluster in | string | n/a | yes |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
 | subnetwork | The subnetwork created to host the cluster in | string | n/a | yes |
@@ -22,11 +22,21 @@ This example illustrates how to create a VPC and a simple cluster.
 |------|-------------|
 | ca\_certificate |  |
 | client\_token |  |
+| cluster\_name | Cluster name |
+| ip\_range\_pods | The secondary IP range used for pods |
+| ip\_range\_services | The secondary IP range used for services |
 | kubernetes\_endpoint |  |
+| location |  |
+| master\_kubernetes\_version | The master Kubernetes version |
+| network |  |
 | network\_name | The name of the VPC being created |
+| project\_id |  |
+| region |  |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | subnet\_names | The name of the subnet being created |
 | subnet\_secondary\_ranges | The secondary ranges associated with the subnet |
+| subnetwork |  |
+| zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index 1f8186e853..ebb4ddb290 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -25,7 +25,7 @@ provider "google" {
 
 module "gcp-network" {
   source       = "terraform-google-modules/network/google"
-  version = "~> 1.4.0"
+  version      = "~> 1.4.0"
   project_id   = var.project_id
   network_name = var.network
 
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index d0d6b3b91d..23caab9b52 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -1,17 +1,18 @@
-
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 resource "random_string" "suffix" {
   length  = 4
@@ -19,10 +20,10 @@ resource "random_string" "suffix" {
   upper   = false
 }
 locals {
-  network = "gke-network-${random_string.suffix.result}"
-  subnetwork = "gke-subnetwork-${random_string.suffix.result}"
-  ip_range_pods="gke-ip-range-pods-${random_string.suffix.result}"
-  ip_range_services="gke-ip-range-svc-${random_string.suffix.result}"
+  network           = "gke-network-${random_string.suffix.result}"
+  subnetwork        = "gke-subnetwork-${random_string.suffix.result}"
+  ip_range_pods     = "gke-ip-range-pods-${random_string.suffix.result}"
+  ip_range_services = "gke-ip-range-svc-${random_string.suffix.result}"
 }
 module "example" {
   source = "../../../examples/simple_regional_with_networking"
@@ -35,4 +36,4 @@ module "example" {
   ip_range_pods                  = local.ip_range_pods
   ip_range_services              = local.ip_range_services
   compute_engine_service_account = var.compute_engine_service_account
-}
\ No newline at end of file
+}
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
index 31da62d519..ff11179f95 100644
--- a/test/fixtures/simple_regional_with_networking/outputs.tf
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -70,4 +70,4 @@ output "ip_range_pods_name" {
 output "ip_range_services_name" {
   description = "The secondary range name for services"
   value       = local.ip_range_services
-}
\ No newline at end of file
+}
diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf
index bc35e84fee..5b2187af58 100644
--- a/test/fixtures/simple_regional_with_networking/variables.tf
+++ b/test/fixtures/simple_regional_with_networking/variables.tf
@@ -25,7 +25,7 @@ variable "cluster_name_suffix" {
 
 variable "region" {
   description = "The region to host the cluster in"
-   default     = "us-east4"
+  default     = "us-east4"
 }
 
 
diff --git a/test/integration/simple_regional_with_networking/controls/gcloud.rb b/test/integration/simple_regional_with_networking/controls/gcloud.rb
index e6152ec2a0..e6bbcfc047 100644
--- a/test/integration/simple_regional_with_networking/controls/gcloud.rb
+++ b/test/integration/simple_regional_with_networking/controls/gcloud.rb
@@ -169,4 +169,4 @@
       end
     end
   end
-end
\ No newline at end of file
+end
diff --git a/test/integration/simple_regional_with_networking/controls/network.rb b/test/integration/simple_regional_with_networking/controls/network.rb
index db356efe28..a17ce74663 100644
--- a/test/integration/simple_regional_with_networking/controls/network.rb
+++ b/test/integration/simple_regional_with_networking/controls/network.rb
@@ -15,18 +15,14 @@
 project_id = attribute('project_id')
 network_name = attribute('network_name')
 subnet_name = attribute('subnet_name')
-
 control "network" do
     title "gcp network configuration"
-    
     describe google_compute_network(
       project: project_id,
       name: network_name
     ) do
       it { should exist }
       its ('subnetworks.count') { should eq 1 }
-      its ('subnetworks.first') { should match subnet_name}
+      its ('subnetworks.first') { should match subnet_name }
     end
   end
-
-  
\ No newline at end of file
diff --git a/test/integration/simple_regional_with_networking/controls/subnet.rb b/test/integration/simple_regional_with_networking/controls/subnet.rb
index f1800e3654..f88d46355b 100644
--- a/test/integration/simple_regional_with_networking/controls/subnet.rb
+++ b/test/integration/simple_regional_with_networking/controls/subnet.rb
@@ -18,14 +18,11 @@
 region = attribute('region')
 ip_range_pods_name = attribute('ip_range_pods_name')
 ip_range_services_name = attribute('ip_range_services_name')
-
 control "subnet" do
     title "gcp subnetwork configuration"
-  
     describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do
       its(:exit_status) { should eq 0 }
       its(:stderr) { should eq '' }
-  
       let(:data) do
         if subject.exit_status == 0
           JSON.parse(subject.stdout)
@@ -46,4 +43,4 @@
         )
       end
     end
-  end
\ No newline at end of file
+  end

From ccec659cee290b6bcbb05943a808e07caec86c92 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Thu, 17 Oct 2019 17:48:29 +0800
Subject: [PATCH 15/90] Release Channels

---
 autogen/cluster.tf   | 10 ++++++++++
 autogen/main.tf      |  4 ++++
 autogen/outputs.tf   |  9 +++++++++
 autogen/variables.tf | 11 +++++++++++
 4 files changed, 34 insertions(+)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 296b2818df..3b9b0fac08 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -45,6 +45,16 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+{% if beta_cluster %}
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
+  }
+{% endif %}
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/autogen/main.tf b/autogen/main.tf
index afbd7bf8c1..9ed92d9741 100644
--- a/autogen/main.tf
+++ b/autogen/main.tf
@@ -48,6 +48,10 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+{% if beta_cluster %}
+  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
+{% endif %}
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/autogen/outputs.tf b/autogen/outputs.tf
index ff8eab1bef..2f3e73fbb0 100644
--- a/autogen/outputs.tf
+++ b/autogen/outputs.tf
@@ -150,4 +150,13 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
+output "release_channel_enabled" {
+  description = "Whether release channel is enabled"
+  value       = var.enable_release_channel
+}
+
+output "release_channel" {
+  description = "The release channel of this cluster"
+  value       = var.release_channel
+}
 {% endif %}
diff --git a/autogen/variables.tf b/autogen/variables.tf
index af446afff8..981c18d672 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -417,4 +417,15 @@ variable "authenticator_security_group" {
   default     = null
 }
 
+variable "enable_release_channel" {
+  type        = bool
+  description = "(Beta) Whether release channel is configured for this cluster."
+  default     = false
+}
+
+variable "release_channel" {
+  type        = string
+  description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
+  default     = "UNSPECIFIED"
+}
 {% endif %}

From 5d829c9e3cf38ca0fa2329999ed470e93fcd371c Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Thu, 17 Oct 2019 17:54:08 +0800
Subject: [PATCH 16/90] Generate from template

---
 cluster.tf                                         |  1 +
 main.tf                                            |  1 +
 .../beta-private-cluster-update-variant/README.md  |  6 +++++-
 .../beta-private-cluster-update-variant/cluster.tf |  8 ++++++++
 .../beta-private-cluster-update-variant/main.tf    |  5 ++++-
 .../networks.tf                                    |  2 +-
 .../beta-private-cluster-update-variant/outputs.tf |  9 +++++++++
 .../variables.tf                                   | 14 +++++++++++++-
 modules/beta-private-cluster/README.md             |  3 +++
 modules/beta-private-cluster/cluster.tf            |  8 ++++++++
 modules/beta-private-cluster/main.tf               |  2 ++
 modules/beta-private-cluster/outputs.tf            |  9 +++++++++
 modules/beta-private-cluster/variables.tf          | 11 +++++++++++
 modules/beta-public-cluster/README.md              |  3 +++
 modules/beta-public-cluster/cluster.tf             |  8 ++++++++
 modules/beta-public-cluster/main.tf                |  2 ++
 modules/beta-public-cluster/outputs.tf             |  9 +++++++++
 modules/beta-public-cluster/variables.tf           | 11 +++++++++++
 modules/private-cluster-update-variant/README.md   |  2 +-
 modules/private-cluster-update-variant/cluster.tf  |  1 +
 modules/private-cluster-update-variant/main.tf     |  4 +++-
 modules/private-cluster-update-variant/networks.tf |  2 +-
 .../private-cluster-update-variant/variables.tf    |  3 ++-
 modules/private-cluster/cluster.tf                 |  1 +
 modules/private-cluster/main.tf                    |  1 +
 25 files changed, 118 insertions(+), 8 deletions(-)

diff --git a/cluster.tf b/cluster.tf
index ffdb27b0fc..c9519113a8 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/main.tf b/main.tf
index a9e1c15810..1090227fd8 100644
--- a/main.tf
+++ b/main.tf
@@ -45,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 0fc0068e96..00bd72d7a2 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -153,6 +153,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -188,8 +189,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
@@ -223,6 +225,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_pools\_versions | List of node pools versions |
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index cf1def945d..7960a90124 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -41,6 +41,14 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
+  }
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index fc38644871..760209d6df 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google-beta
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
@@ -43,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf
index 14ea500e03..2456654130 100644
--- a/modules/beta-private-cluster-update-variant/networks.tf
+++ b/modules/beta-private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google-beta
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf
index 4153960069..31a22b2175 100644
--- a/modules/beta-private-cluster-update-variant/outputs.tf
+++ b/modules/beta-private-cluster-update-variant/outputs.tf
@@ -149,3 +149,12 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
+output "release_channel_enabled" {
+  description = "Whether release channel is enabled"
+  value       = var.enable_release_channel
+}
+
+output "release_channel" {
+  description = "The release channel of this cluster"
+  value       = var.release_channel
+}
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 9a869a830f..90b01e30c5 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -405,3 +406,14 @@ variable "authenticator_security_group" {
   default     = null
 }
 
+variable "enable_release_channel" {
+  type        = bool
+  description = "(Beta) Whether release channel is configured for this cluster."
+  default     = false
+}
+
+variable "release_channel" {
+  type        = string
+  description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
+  default     = "UNSPECIFIED"
+}
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 425ef67fa1..341013c059 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -153,6 +153,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -224,6 +225,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_pools\_versions | List of node pools versions |
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index c481c69a35..a3aa0b12ef 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -41,6 +41,14 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
+  }
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
index 63bf31ac78..760209d6df 100644
--- a/modules/beta-private-cluster/main.tf
+++ b/modules/beta-private-cluster/main.tf
@@ -44,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf
index 4153960069..31a22b2175 100644
--- a/modules/beta-private-cluster/outputs.tf
+++ b/modules/beta-private-cluster/outputs.tf
@@ -149,3 +149,12 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
+output "release_channel_enabled" {
+  description = "Whether release channel is enabled"
+  value       = var.enable_release_channel
+}
+
+output "release_channel" {
+  description = "The release channel of this cluster"
+  value       = var.release_channel
+}
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index ee2d5b5556..2c9dfb1e4d 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -412,3 +412,14 @@ variable "authenticator_security_group" {
   default     = null
 }
 
+variable "enable_release_channel" {
+  type        = bool
+  description = "(Beta) Whether release channel is configured for this cluster."
+  default     = false
+}
+
+variable "release_channel" {
+  type        = string
+  description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
+  default     = "UNSPECIFIED"
+}
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 73cf4ea4c5..399e583d6c 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -145,6 +145,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -215,6 +216,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_pools\_versions | List of node pools versions |
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index a264e932b9..fd37709fd0 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -41,6 +41,14 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
+  }
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf
index f0477f9498..ec0afeda3b 100644
--- a/modules/beta-public-cluster/main.tf
+++ b/modules/beta-public-cluster/main.tf
@@ -44,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf
index 4153960069..31a22b2175 100644
--- a/modules/beta-public-cluster/outputs.tf
+++ b/modules/beta-public-cluster/outputs.tf
@@ -149,3 +149,12 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
+output "release_channel_enabled" {
+  description = "Whether release channel is enabled"
+  value       = var.enable_release_channel
+}
+
+output "release_channel" {
+  description = "The release channel of this cluster"
+  value       = var.release_channel
+}
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 90008d54bd..f004155988 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -388,3 +388,14 @@ variable "authenticator_security_group" {
   default     = null
 }
 
+variable "enable_release_channel" {
+  type        = bool
+  description = "(Beta) Whether release channel is configured for this cluster."
+  default     = false
+}
+
+variable "release_channel" {
+  type        = string
+  description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
+  default     = "UNSPECIFIED"
+}
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index e817361124..8b0a140c4c 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -174,7 +174,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index e8db91a77a..c4342d0d8f 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index bfe746401c..aba5e2d79f 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
@@ -44,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf
index a382073dc0..aae034eee5 100644
--- a/modules/private-cluster-update-variant/networks.tf
+++ b/modules/private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 8008e08975..28b744d868 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index 412e8295ed..ee8f7d433f 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
   min_master_version = local.master_version
 
diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
index 2bd1c40d14..aba5e2d79f 100644
--- a/modules/private-cluster/main.tf
+++ b/modules/private-cluster/main.tf
@@ -45,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

From b63658ef98e8e1c5477fadbb18de0a836163b748 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Mon, 21 Oct 2019 10:10:06 +0800
Subject: [PATCH 17/90] Merge from master

---
 modules/beta-private-cluster-update-variant/README.md    | 4 ++++
 modules/beta-private-cluster-update-variant/sa.tf        | 2 +-
 modules/beta-private-cluster-update-variant/variables.tf | 6 ++++++
 modules/beta-private-cluster/README.md                   | 1 +
 modules/beta-public-cluster/README.md                    | 1 +
 modules/private-cluster-update-variant/README.md         | 4 ++++
 modules/private-cluster-update-variant/sa.tf             | 2 +-
 modules/private-cluster-update-variant/variables.tf      | 6 ++++++
 scripts/wait-for-cluster.sh                              | 6 +-----
 9 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 00bd72d7a2..c2f9aeffd4 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -191,6 +191,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
@@ -262,6 +263,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/beta-private-cluster-update-variant/sa.tf
+++ b/modules/beta-private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 90b01e30c5..2c9dfb1e4d 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -268,6 +268,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 341013c059..29c5146687 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -192,6 +192,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 399e583d6c..bfb6045521 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -183,6 +183,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index 8b0a140c4c..e0ef130d3e 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -176,6 +176,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
@@ -237,6 +238,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/private-cluster-update-variant/sa.tf
+++ b/modules/private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 28b744d868..a425c13d9a 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -258,6 +258,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 37f0176ec7..6ff3253d58 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2019 Google LLC
+# Copyright 2018 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,10 +15,6 @@
 
 set -e
 
-if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
-fi
-
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

From e90d20bac2a364ebdaec067273ec2c676790a977 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Mon, 21 Oct 2019 11:05:51 +0300
Subject: [PATCH 18/90] Created test for sandbox (Fix #252)

---
 .kitchen.yml                                  |   7 ++
 examples/simple_regional_beta/README.md       |  11 +-
 examples/simple_regional_beta/main.tf         |  35 +++---
 examples/simple_regional_beta/test_outputs.tf |   4 -
 examples/simple_regional_beta/variables.tf    |  33 +++++-
 test/fixtures/sandbox_enabled/example.tf      |  40 +++++++
 test/fixtures/sandbox_enabled/network.tf      |  48 +++++++++
 test/fixtures/sandbox_enabled/outputs.tf      |   1 +
 test/fixtures/sandbox_enabled/variables.tf    |   1 +
 .../sandbox_enabled/controls/gcloud.rb        | 102 ++++++++++++++++++
 test/integration/sandbox_enabled/inspec.yml   |  17 +++
 11 files changed, 270 insertions(+), 29 deletions(-)
 create mode 100644 test/fixtures/sandbox_enabled/example.tf
 create mode 100644 test/fixtures/sandbox_enabled/network.tf
 create mode 120000 test/fixtures/sandbox_enabled/outputs.tf
 create mode 120000 test/fixtures/sandbox_enabled/variables.tf
 create mode 100644 test/integration/sandbox_enabled/controls/gcloud.rb
 create mode 100644 test/integration/sandbox_enabled/inspec.yml

diff --git a/.kitchen.yml b/.kitchen.yml
index 9f5df5a03e..61b8aae029 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -131,3 +131,10 @@ suites:
       systems:
         - name: workload_metadata_config
           backend: local
+  - name: "sandbox_enabled"
+    driver:
+      root_module_directory: test/fixtures/sandbox_enabled
+    verifier:
+      systems:
+        - name: sandbox_enabled
+          backend: local
diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md
index bd676115b9..02d0dba224 100644
--- a/examples/simple_regional_beta/README.md
+++ b/examples/simple_regional_beta/README.md
@@ -2,8 +2,7 @@
 
 This example illustrates how to create a simple cluster with beta features.
 
-[^]: (autogen_docs_start)
-
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
@@ -11,13 +10,16 @@ This example illustrates how to create a simple cluster with beta features.
 | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no |
 | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
-| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes |
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
 | istio | Boolean to enable / disable Istio | string | `"true"` | no |
 | network | The VPC network to host the cluster in | string | n/a | yes |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
 
 ## Outputs
@@ -27,7 +29,6 @@ This example illustrates how to create a simple cluster with beta features.
 | ca\_certificate |  |
 | client\_token |  |
 | cluster\_name | Cluster name |
-| credentials\_path |  |
 | ip\_range\_pods | The secondary IP range used for pods |
 | ip\_range\_services | The secondary IP range used for services |
 | kubernetes\_endpoint |  |
@@ -40,7 +41,7 @@ This example illustrates how to create a simple cluster with beta features.
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
-[^]: (autogen_docs_end)
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 To provision this example, run the following from within this directory:
 - `terraform init` to get the plugins
diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf
index fc95090ede..b75fdaa613 100644
--- a/examples/simple_regional_beta/main.tf
+++ b/examples/simple_regional_beta/main.tf
@@ -19,25 +19,28 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
-  credentials = file(var.credentials_path)
-  region      = var.region
+  version = "~> 2.12.0"
+  region  = var.region
 }
 
 module "gke" {
-  source                 = "../../modules/beta-public-cluster/"
-  project_id             = var.project_id
-  name                   = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  regional               = true
-  region                 = var.region
-  network                = var.network
-  subnetwork             = var.subnetwork
-  ip_range_pods          = var.ip_range_pods
-  ip_range_services      = var.ip_range_services
-  create_service_account = false
-  service_account        = var.compute_engine_service_account
-  istio                  = var.istio
-  cloudrun               = var.cloudrun
+  source                   = "../../modules/beta-public-cluster/"
+  project_id               = var.project_id
+  name                     = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                 = true
+  region                   = var.region
+  network                  = var.network
+  subnetwork               = var.subnetwork
+  ip_range_pods            = var.ip_range_pods
+  ip_range_services        = var.ip_range_services
+  create_service_account   = false
+  service_account          = var.compute_engine_service_account
+  istio                    = var.istio
+  cloudrun                 = var.cloudrun
+  node_metadata            = var.node_metadata
+  sandbox_enabled          = var.sandbox_enabled
+  remove_default_node_pool = var.remove_default_node_pool
+  node_pools               = var.node_pools
 }
 
 data "google_client_config" "default" {
diff --git a/examples/simple_regional_beta/test_outputs.tf b/examples/simple_regional_beta/test_outputs.tf
index f250fef192..e64c40e477 100644
--- a/examples/simple_regional_beta/test_outputs.tf
+++ b/examples/simple_regional_beta/test_outputs.tf
@@ -21,10 +21,6 @@ output "project_id" {
   value = var.project_id
 }
 
-output "credentials_path" {
-  value = var.credentials_path
-}
-
 output "region" {
   value = module.gke.region
 }
diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf
index 1da408a790..ed16642774 100644
--- a/examples/simple_regional_beta/variables.tf
+++ b/examples/simple_regional_beta/variables.tf
@@ -18,10 +18,6 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "credentials_path" {
-  description = "The path to the GCP credentials JSON file"
-}
-
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""
@@ -60,3 +56,32 @@ variable "cloudrun" {
   description = "Boolean to enable / disable CloudRun"
   default     = true
 }
+
+variable "node_metadata" {
+  description = "Specifies how node metadata is exposed to the workload running on the node"
+  default     = "SECURE"
+  type        = string
+}
+
+variable "sandbox_enabled" {
+  type        = bool
+  description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)."
+  default     = false
+}
+
+variable "remove_default_node_pool" {
+  type        = bool
+  description = "Remove default node pool while setting up the cluster"
+  default     = false
+}
+
+variable "node_pools" {
+  type        = list(map(string))
+  description = "List of maps containing node pools"
+
+  default = [
+    {
+      name = "default-node-pool"
+    },
+  ]
+}
diff --git a/test/fixtures/sandbox_enabled/example.tf b/test/fixtures/sandbox_enabled/example.tf
new file mode 100644
index 0000000000..05b7edfd9e
--- /dev/null
+++ b/test/fixtures/sandbox_enabled/example.tf
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "example" {
+  source = "../../../examples/simple_regional_beta"
+
+  project_id                     = var.project_id
+  cluster_name_suffix            = "-${random_string.suffix.result}"
+  region                         = var.region
+  network                        = google_compute_network.main.name
+  subnetwork                     = google_compute_subnetwork.main.name
+  ip_range_pods                  = google_compute_subnetwork.main.secondary_ip_range[0].range_name
+  ip_range_services              = google_compute_subnetwork.main.secondary_ip_range[1].range_name
+  compute_engine_service_account = var.compute_engine_service_account
+  istio                          = false
+  cloudrun                       = false
+  node_metadata                  = "UNSPECIFIED"
+  sandbox_enabled                = true
+  remove_default_node_pool       = true
+
+  node_pools = [
+    {
+      name       = "default-node-pool"
+      image_type = "COS_CONTAINERD"
+    },
+  ]
+}
diff --git a/test/fixtures/sandbox_enabled/network.tf b/test/fixtures/sandbox_enabled/network.tf
new file mode 100644
index 0000000000..5d34d43748
--- /dev/null
+++ b/test/fixtures/sandbox_enabled/network.tf
@@ -0,0 +1,48 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}
+
+provider "google" {
+  project = var.project_id
+}
+
+resource "google_compute_network" "main" {
+  name                    = "cft-gke-test-${random_string.suffix.result}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "main" {
+  name          = "cft-gke-test-${random_string.suffix.result}"
+  ip_cidr_range = "10.0.0.0/17"
+  region        = var.region
+  network       = google_compute_network.main.self_link
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-pods-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.0.0/18"
+  }
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-services-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.64.0/18"
+  }
+}
+
diff --git a/test/fixtures/sandbox_enabled/outputs.tf b/test/fixtures/sandbox_enabled/outputs.tf
new file mode 120000
index 0000000000..726bdc722f
--- /dev/null
+++ b/test/fixtures/sandbox_enabled/outputs.tf
@@ -0,0 +1 @@
+../shared/outputs.tf
\ No newline at end of file
diff --git a/test/fixtures/sandbox_enabled/variables.tf b/test/fixtures/sandbox_enabled/variables.tf
new file mode 120000
index 0000000000..c113c00a3d
--- /dev/null
+++ b/test/fixtures/sandbox_enabled/variables.tf
@@ -0,0 +1 @@
+../shared/variables.tf
\ No newline at end of file
diff --git a/test/integration/sandbox_enabled/controls/gcloud.rb b/test/integration/sandbox_enabled/controls/gcloud.rb
new file mode 100644
index 0000000000..eb0ffdaf46
--- /dev/null
+++ b/test/integration/sandbox_enabled/controls/gcloud.rb
@@ -0,0 +1,102 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project_id = attribute('project_id')
+location = attribute('location')
+cluster_name = attribute('cluster_name')
+
+control "gcloud" do
+  title "Google Compute Engine GKE configuration"
+  describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let!(:data) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+
+    describe "cluster" do
+      it "is running" do
+        expect(data['status']).to eq 'RUNNING'
+      end
+
+      it "is regional" do
+        expect(data['location']).to match(/^.*[1-9]$/)
+      end
+
+      it "uses public nodes and master endpoint" do
+        expect(data['privateClusterConfig']).to eq nil
+      end
+
+      it "has the expected addon settings" do
+        expect(data['addonsConfig']).to eq({
+          "horizontalPodAutoscaling" => {},
+          "httpLoadBalancing" => {},
+          "kubernetesDashboard" => {
+            "disabled" => true,
+          },
+          "networkPolicyConfig" => {
+            "disabled" => true,
+          },
+        })
+      end
+    end
+
+    describe "node pool" do
+      let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } }
+
+      it "is the expected image type" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "imageType" => "COS_CONTAINERD",
+            ),
+          )
+        )
+      end
+
+      it "has the expected labels" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "labels" => including(
+                "cluster_name" => cluster_name,
+                "node_pool" => "default-node-pool",
+		"sandbox.gke.io/runtime" => "gvisor",
+              ),
+            ),
+          )
+        )
+      end
+
+      it "has the expected network tags" do
+        expect(node_pools).to include(
+          including(
+            "config" => including(
+              "tags" => match_array([
+                "gke-#{cluster_name}",
+                "gke-#{cluster_name}-default-node-pool",
+              ]),
+            ),
+          )
+        )
+      end
+
+    end
+  end
+end
diff --git a/test/integration/sandbox_enabled/inspec.yml b/test/integration/sandbox_enabled/inspec.yml
new file mode 100644
index 0000000000..0454937a36
--- /dev/null
+++ b/test/integration/sandbox_enabled/inspec.yml
@@ -0,0 +1,17 @@
+name: sandbox_enabled
+attributes:
+  - name: project_id
+    required: true
+    type: string
+  - name: location
+    required: true
+    type: string
+  - name: cluster_name
+    required: true
+    type: string
+  - name: kubernetes_endpoint
+    required: true
+    type: string
+  - name: client_token
+    required: true
+    type: string

From 8a2fddc70bd5631195714a1f671ab4b9c2114392 Mon Sep 17 00:00:00 2001
From: omazin <osmansimf@rambler.ru>
Date: Mon, 21 Oct 2019 15:52:33 +0300
Subject: [PATCH 19/90] [wait-for-cluster.sh] Fix #284.

---
 autogen/scripts/wait-for-cluster.sh                             | 2 +-
 .../scripts/wait-for-cluster.sh                                 | 2 +-
 modules/beta-private-cluster/scripts/wait-for-cluster.sh        | 2 +-
 modules/beta-public-cluster/scripts/wait-for-cluster.sh         | 2 +-
 .../private-cluster-update-variant/scripts/wait-for-cluster.sh  | 2 +-
 modules/private-cluster/scripts/wait-for-cluster.sh             | 2 +-
 scripts/wait-for-cluster.sh                                     | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/autogen/scripts/wait-for-cluster.sh b/autogen/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/autogen/scripts/wait-for-cluster.sh
+++ b/autogen/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/modules/beta-private-cluster/scripts/wait-for-cluster.sh b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/modules/beta-private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/modules/beta-public-cluster/scripts/wait-for-cluster.sh b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/modules/beta-public-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/modules/private-cluster/scripts/wait-for-cluster.sh b/modules/private-cluster/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/modules/private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 37f0176ec7..7bf9919b8f 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -16,7 +16,7 @@
 set -e
 
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 PROJECT=$1

From ebabcd0870f72d16dc48f25218731a19b93239e7 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Tue, 22 Oct 2019 21:54:38 -0500
Subject: [PATCH 20/90] add defaults, remove svc account, remove local

---
 examples/simple_regional_with_networking/README.md | 12 +++++-------
 examples/simple_regional_with_networking/main.tf   |  9 ++-------
 .../simple_regional_with_networking/variables.tf   | 14 +++++---------
 3 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index f39675063d..33a19683d0 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -7,14 +7,12 @@ This example illustrates how to create a VPC and a simple cluster.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
-| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
-| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
-| network | The VPC network created to host the cluster in | string | n/a | yes |
+| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no |
+| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no |
+| network | The VPC network created to host the cluster in | string | `"gke-network"` | no |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
-| region | The region to host the cluster in | string | n/a | yes |
-| subnetwork | The subnetwork created to host the cluster in | string | n/a | yes |
+| region | The region to host the cluster in | string | `"us-central1"` | no |
+| subnetwork | The subnetwork created to host the cluster in | string | `"gke-subnet"` | no |
 
 ## Outputs
 
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index ebb4ddb290..8bb70d6db9 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -14,10 +14,6 @@
  * limitations under the License.
  */
 
-locals {
-  cluster_type = "simple-regional"
-}
-
 provider "google" {
   version = "~> 2.12.0"
   region  = var.region
@@ -54,15 +50,14 @@ module "gcp-network" {
 module "gke" {
   source                 = "../../"
   project_id             = var.project_id
-  name                   = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  name                   = "simple-regional-cluster"
   regional               = true
   region                 = var.region
   network                = module.gcp-network.network_name
   subnetwork             = module.gcp-network.subnets_names[0]
   ip_range_pods          = var.ip_range_pods
   ip_range_services      = var.ip_range_services
-  create_service_account = false
-  service_account        = var.compute_engine_service_account
+  create_service_account = true
 }
 
 data "google_client_config" "default" {
diff --git a/examples/simple_regional_with_networking/variables.tf b/examples/simple_regional_with_networking/variables.tf
index 5c36a81c16..6896accbe4 100644
--- a/examples/simple_regional_with_networking/variables.tf
+++ b/examples/simple_regional_with_networking/variables.tf
@@ -18,32 +18,28 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "cluster_name_suffix" {
-  description = "A suffix to append to the default cluster name"
-  default     = ""
-}
-
 variable "region" {
   description = "The region to host the cluster in"
+  default     = "us-central1"
 }
 
 variable "network" {
   description = "The VPC network created to host the cluster in"
+  default     = "gke-network"
 }
 
 variable "subnetwork" {
   description = "The subnetwork created to host the cluster in"
+  default     = "gke-subnet"
 }
 
 variable "ip_range_pods" {
   description = "The secondary ip range to use for pods"
+  default     = "ip-range-pods"
 }
 
 variable "ip_range_services" {
   description = "The secondary ip range to use for pods"
-}
-
-variable "compute_engine_service_account" {
-  description = "Service account to associate to the nodes in the cluster"
+  default     = "ip-range-scv"
 }
 

From 5a4f6ce7429e1e1f29f91f3c412fe4dc185456e7 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Tue, 22 Oct 2019 22:22:35 -0500
Subject: [PATCH 21/90] fix fixtures

---
 examples/simple_regional_with_networking/README.md   |  1 +
 examples/simple_regional_with_networking/main.tf     |  2 +-
 .../simple_regional_with_networking/variables.tf     |  5 +++++
 .../simple_regional_with_networking/example.tf       | 12 +-----------
 .../simple_regional_with_networking/outputs.tf       |  8 ++++----
 .../simple_regional_with_networking/variables.tf     | 10 ----------
 6 files changed, 12 insertions(+), 26 deletions(-)

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index 33a19683d0..e24ee546fb 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -7,6 +7,7 @@ This example illustrates how to create a VPC and a simple cluster.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no |
 | ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no |
 | network | The VPC network created to host the cluster in | string | `"gke-network"` | no |
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index 8bb70d6db9..fc53436f2d 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -50,7 +50,7 @@ module "gcp-network" {
 module "gke" {
   source                 = "../../"
   project_id             = var.project_id
-  name                   = "simple-regional-cluster"
+  name                   = "simple-regional-cluster-${var.cluster_name_suffix}"
   regional               = true
   region                 = var.region
   network                = module.gcp-network.network_name
diff --git a/examples/simple_regional_with_networking/variables.tf b/examples/simple_regional_with_networking/variables.tf
index 6896accbe4..e9045a5c0d 100644
--- a/examples/simple_regional_with_networking/variables.tf
+++ b/examples/simple_regional_with_networking/variables.tf
@@ -18,6 +18,11 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
+variable "cluster_name_suffix" {
+  description = "A suffix to append to the default cluster name"
+  default     = ""
+}
+
 variable "region" {
   description = "The region to host the cluster in"
   default     = "us-central1"
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index 23caab9b52..01506b034c 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -19,21 +19,11 @@ resource "random_string" "suffix" {
   special = false
   upper   = false
 }
-locals {
-  network           = "gke-network-${random_string.suffix.result}"
-  subnetwork        = "gke-subnetwork-${random_string.suffix.result}"
-  ip_range_pods     = "gke-ip-range-pods-${random_string.suffix.result}"
-  ip_range_services = "gke-ip-range-svc-${random_string.suffix.result}"
-}
+
 module "example" {
   source = "../../../examples/simple_regional_with_networking"
 
   project_id                     = var.project_id
   cluster_name_suffix            = "-${random_string.suffix.result}"
   region                         = var.region
-  network                        = local.network
-  subnetwork                     = local.subnetwork
-  ip_range_pods                  = local.ip_range_pods
-  ip_range_services              = local.ip_range_services
-  compute_engine_service_account = var.compute_engine_service_account
 }
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
index ff11179f95..fd0f7651a6 100644
--- a/test/fixtures/simple_regional_with_networking/outputs.tf
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -49,12 +49,12 @@ output "service_account" {
 
 output "network_name" {
   description = "The name of the VPC being created"
-  value       = local.network
+  value       = var.network
 }
 
 output "subnet_name" {
   description = "The name of the subnet being created"
-  value       = local.subnetwork
+  value       = var.subnetwork
 }
 
 output "region" {
@@ -64,10 +64,10 @@ output "region" {
 
 output "ip_range_pods_name" {
   description = "The secondary range name for pods"
-  value       = local.ip_range_pods
+  value       = var.ip_range_pods
 }
 
 output "ip_range_services_name" {
   description = "The secondary range name for services"
-  value       = local.ip_range_services
+  value       = var.ip_range_services
 }
diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf
index 5b2187af58..b0ba1f508f 100644
--- a/test/fixtures/simple_regional_with_networking/variables.tf
+++ b/test/fixtures/simple_regional_with_networking/variables.tf
@@ -18,18 +18,8 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "cluster_name_suffix" {
-  description = "A suffix to append to the default cluster name"
-  default     = ""
-}
-
 variable "region" {
   description = "The region to host the cluster in"
   default     = "us-east4"
 }
 
-
-variable "compute_engine_service_account" {
-  description = "Service account to associate to the nodes in the cluster"
-}
-

From 258543e21e4e40bc82e568413599a0d143d784dd Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Tue, 22 Oct 2019 23:04:30 -0500
Subject: [PATCH 22/90] fix tests

---
 .../example.tf                                | 10 +++++++---
 .../outputs.tf                                |  8 ++++----
 .../variables.tf                              | 19 +++++++++++++++++++
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index 01506b034c..59f6c27ba2 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -23,7 +23,11 @@ resource "random_string" "suffix" {
 module "example" {
   source = "../../../examples/simple_regional_with_networking"
 
-  project_id                     = var.project_id
-  cluster_name_suffix            = "-${random_string.suffix.result}"
-  region                         = var.region
+  project_id          = var.project_id
+  cluster_name_suffix = "-${random_string.suffix.result}"
+  region              = var.region
+  network             = "${var.network}-${random_string.suffix.result}"
+  subnetwork          = "${var.subnetwork}-${random_string.suffix.result}"
+  ip_range_pods       = "${var.ip_range_pods}-${random_string.suffix.result}"
+  ip_range_services   = "${var.ip_range_services}-${random_string.suffix.result}"
 }
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
index fd0f7651a6..2a13237983 100644
--- a/test/fixtures/simple_regional_with_networking/outputs.tf
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -49,12 +49,12 @@ output "service_account" {
 
 output "network_name" {
   description = "The name of the VPC being created"
-  value       = var.network
+  value       = module.example.network
 }
 
 output "subnet_name" {
   description = "The name of the subnet being created"
-  value       = var.subnetwork
+  value       = module.example.subnetwork
 }
 
 output "region" {
@@ -64,10 +64,10 @@ output "region" {
 
 output "ip_range_pods_name" {
   description = "The secondary range name for pods"
-  value       = var.ip_range_pods
+  value       = module.example.ip_range_pods
 }
 
 output "ip_range_services_name" {
   description = "The secondary range name for services"
-  value       = var.ip_range_services
+  value       = module.example.ip_range_services
 }
diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf
index b0ba1f508f..989451fe32 100644
--- a/test/fixtures/simple_regional_with_networking/variables.tf
+++ b/test/fixtures/simple_regional_with_networking/variables.tf
@@ -23,3 +23,22 @@ variable "region" {
   default     = "us-east4"
 }
 
+variable "network" {
+  description = "The VPC network created to host the cluster in"
+  default     = "gke-network"
+}
+
+variable "subnetwork" {
+  description = "The subnetwork created to host the cluster in"
+  default     = "gke-subnet"
+}
+
+variable "ip_range_pods" {
+  description = "The secondary ip range to use for pods"
+  default     = "ip-range-pods"
+}
+
+variable "ip_range_services" {
+  description = "The secondary ip range to use for pods"
+  default     = "ip-range-scv"
+}

From b050668eecae7c3e8d2f48da1d014f53bf941fb7 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Wed, 23 Oct 2019 00:33:58 -0500
Subject: [PATCH 23/90] add ci yaml

---
 test/ci/simple-regional-with-networking.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 test/ci/simple-regional-with-networking.yml

diff --git a/test/ci/simple-regional-with-networking.yml b/test/ci/simple-regional-with-networking.yml
new file mode 100644
index 0000000000..022044bc8e
--- /dev/null
+++ b/test/ci/simple-regional-with-networking.yml
@@ -0,0 +1,18 @@
+---
+
+platform: linux
+
+inputs:
+- name: pull-request
+  path: terraform-google-kubernetes-engine
+
+run:
+  path: make
+  args: ['test_integration']
+  dir: terraform-google-kubernetes-engine
+
+params:
+  SUITE: "imple-regional-with-networking-local"
+  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
+  REGION: "us-east4"
+  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'

From 6ee11ce0e01b26667216fd553e0ed71ac4f787c6 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Wed, 23 Oct 2019 00:34:53 -0500
Subject: [PATCH 24/90] add ci yaml fix

---
 test/ci/simple-regional-with-networking.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/ci/simple-regional-with-networking.yml b/test/ci/simple-regional-with-networking.yml
index 022044bc8e..68ba8c38cd 100644
--- a/test/ci/simple-regional-with-networking.yml
+++ b/test/ci/simple-regional-with-networking.yml
@@ -12,7 +12,7 @@ run:
   dir: terraform-google-kubernetes-engine
 
 params:
-  SUITE: "imple-regional-with-networking-local"
+  SUITE: "simple-regional-with-networking-local"
   COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
   REGION: "us-east4"
   ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'

From 53ec7a99287ba00838a9811af8e723f6607fec98 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Wed, 23 Oct 2019 10:49:12 +0300
Subject: [PATCH 25/90] Fix logic of skip_provisioners behavior

---
 README.md                                          |  2 +-
 autogen/README.md                                  |  4 +++-
 autogen/cluster.tf                                 |  2 +-
 autogen/dns.tf                                     |  2 +-
 autogen/variables.tf                               |  2 +-
 cluster.tf                                         |  2 +-
 dns.tf                                             |  2 +-
 .../beta-private-cluster-update-variant/README.md  |  7 ++++++-
 .../beta-private-cluster-update-variant/cluster.tf |  1 +
 modules/beta-private-cluster-update-variant/dns.tf |  2 +-
 .../beta-private-cluster-update-variant/main.tf    |  3 ++-
 .../networks.tf                                    |  2 +-
 modules/beta-private-cluster-update-variant/sa.tf  |  2 +-
 .../variables.tf                                   | 14 +++++++++++++-
 modules/beta-private-cluster/README.md             |  2 +-
 modules/beta-private-cluster/cluster.tf            |  2 +-
 modules/beta-private-cluster/dns.tf                |  2 +-
 modules/beta-private-cluster/variables.tf          |  2 +-
 modules/beta-public-cluster/README.md              |  2 +-
 modules/beta-public-cluster/cluster.tf             |  2 +-
 modules/beta-public-cluster/dns.tf                 |  2 +-
 modules/beta-public-cluster/variables.tf           |  2 +-
 modules/private-cluster-update-variant/README.md   |  7 ++++++-
 modules/private-cluster-update-variant/cluster.tf  |  1 +
 modules/private-cluster-update-variant/dns.tf      |  2 +-
 modules/private-cluster-update-variant/main.tf     |  3 ++-
 modules/private-cluster-update-variant/networks.tf |  2 +-
 modules/private-cluster-update-variant/sa.tf       |  2 +-
 .../private-cluster-update-variant/variables.tf    | 14 +++++++++++++-
 modules/private-cluster/README.md                  |  2 +-
 modules/private-cluster/cluster.tf                 |  2 +-
 modules/private-cluster/dns.tf                     |  2 +-
 modules/private-cluster/variables.tf               |  2 +-
 scripts/wait-for-cluster.sh                        |  6 +-----
 variables.tf                                       |  2 +-
 35 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 8c9a63fdab..15f6aff13b 100644
--- a/README.md
+++ b/README.md
@@ -170,7 +170,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/autogen/README.md b/autogen/README.md
index dc0b63b003..a0655d62f6 100644
--- a/autogen/README.md
+++ b/autogen/README.md
@@ -195,12 +195,14 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index c394945f2d..44d1f28639 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -433,7 +433,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
-  count = var.skip_provisioners ? 1 : 0
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/autogen/dns.tf b/autogen/dns.tf
index 731e010b0d..20c3b25ee9 100644
--- a/autogen/dns.tf
+++ b/autogen/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/autogen/variables.tf b/autogen/variables.tf
index dded3fd6cb..ede2bac6c5 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -313,7 +313,7 @@ variable "cluster_resource_labels" {
 
 variable "skip_provisioners" {
   type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
 {% if private_cluster %}
diff --git a/cluster.tf b/cluster.tf
index 40b5559703..58023049ac 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -227,7 +227,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
-  count = var.skip_provisioners ? 1 : 0
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/dns.tf b/dns.tf
index f490c15504..8a581ff68e 100644
--- a/dns.tf
+++ b/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 0fc0068e96..ba84c19cce 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -188,12 +188,14 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
@@ -258,6 +260,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index cf1def945d..db3d13d413 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -401,6 +401,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/beta-private-cluster-update-variant/dns.tf b/modules/beta-private-cluster-update-variant/dns.tf
index b240a23e65..8a581ff68e 100644
--- a/modules/beta-private-cluster-update-variant/dns.tf
+++ b/modules/beta-private-cluster-update-variant/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index fc38644871..63bf31ac78 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google-beta
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf
index 14ea500e03..2456654130 100644
--- a/modules/beta-private-cluster-update-variant/networks.tf
+++ b/modules/beta-private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google-beta
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/beta-private-cluster-update-variant/sa.tf
+++ b/modules/beta-private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 9a869a830f..2585cf4ba8 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -267,6 +268,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
@@ -302,6 +309,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 
 variable "deploy_using_private_endpoint" {
   type        = bool
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index f80dcae85b..d7415a8af6 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -195,7 +195,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index bb19940d1f..efd394a535 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -328,7 +328,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
-  count = var.skip_provisioners ? 1 : 0
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf
index f490c15504..8a581ff68e 100644
--- a/modules/beta-private-cluster/dns.tf
+++ b/modules/beta-private-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index a5bcec8336..2585cf4ba8 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -311,7 +311,7 @@ variable "cluster_resource_labels" {
 
 variable "skip_provisioners" {
   type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
 
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 1d3bdc17b2..419c7d017f 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -186,7 +186,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index e2e46ac862..f3c342e2d4 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -323,7 +323,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
-  count = var.skip_provisioners ? 1 : 0
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf
index f490c15504..8a581ff68e 100644
--- a/modules/beta-public-cluster/dns.tf
+++ b/modules/beta-public-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 32a8eaa33f..aec51dcf34 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -311,7 +311,7 @@ variable "cluster_resource_labels" {
 
 variable "skip_provisioners" {
   type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
 
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index e817361124..fa9cdb8852 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -174,10 +174,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
@@ -237,6 +239,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index e8db91a77a..c447c39a39 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -305,6 +305,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/private-cluster-update-variant/dns.tf b/modules/private-cluster-update-variant/dns.tf
index b240a23e65..8a581ff68e 100644
--- a/modules/private-cluster-update-variant/dns.tf
+++ b/modules/private-cluster-update-variant/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index bfe746401c..2bd1c40d14 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf
index a382073dc0..aae034eee5 100644
--- a/modules/private-cluster-update-variant/networks.tf
+++ b/modules/private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/private-cluster-update-variant/sa.tf
+++ b/modules/private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 8008e08975..508a4f1b96 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -257,6 +258,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
@@ -292,6 +299,11 @@ variable "cluster_resource_labels" {
   default     = {}
 }
 
+variable "skip_provisioners" {
+  type        = bool
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
+  default     = false
+}
 
 variable "deploy_using_private_endpoint" {
   type        = bool
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 5c01b849db..5465544b82 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -179,7 +179,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
 | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index ab8b275ab1..afb9a633ea 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -232,7 +232,7 @@ resource "google_container_node_pool" "pools" {
 }
 
 resource "null_resource" "wait_for_cluster" {
-  count = var.skip_provisioners ? 1 : 0
+  count = var.skip_provisioners ? 0 : 1
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}"
diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf
index f490c15504..8a581ff68e 100644
--- a/modules/private-cluster/dns.tf
+++ b/modules/private-cluster/dns.tf
@@ -20,7 +20,7 @@
   Delete default kube-dns configmap
  *****************************************/
 resource "null_resource" "delete_default_kube_dns_configmap" {
-  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) || var.skip_provisioners ? 1 : 0
+  count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0
 
   provisioner "local-exec" {
     command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns"
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
index d6381cb297..508a4f1b96 100644
--- a/modules/private-cluster/variables.tf
+++ b/modules/private-cluster/variables.tf
@@ -301,7 +301,7 @@ variable "cluster_resource_labels" {
 
 variable "skip_provisioners" {
   type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
 
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 37f0176ec7..6ff3253d58 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2019 Google LLC
+# Copyright 2018 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,10 +15,6 @@
 
 set -e
 
-if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
-fi
-
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"
diff --git a/variables.tf b/variables.tf
index b7fbfba8cc..58cf1f4685 100644
--- a/variables.tf
+++ b/variables.tf
@@ -301,6 +301,6 @@ variable "cluster_resource_labels" {
 
 variable "skip_provisioners" {
   type        = bool
-  description = "Flag to skip all local-exec provisioners. It breaks down `stub_domains` and `upstream_nameservers` variables functionality."
+  description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }

From bf8a5ef5fed5bc26002970f50c31404195a327d0 Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Tue, 22 Oct 2019 16:05:42 -0400
Subject: [PATCH 26/90] Parallelize execution of Kitchen instances

Running all instances in serial is taking over 3 hours to complete.
---
 build/int.cloudbuild.yaml | 204 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 196 insertions(+), 8 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 85139efe7d..c1efa77057 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -21,18 +21,206 @@ steps:
   - 'TF_VAR_org_id=$_ORG_ID'
   - 'TF_VAR_folder_id=$_FOLDER_ID'
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
-- id: create
+- id: create disable-client-cert-local
+  waitFor:
+    - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
-- id: converge
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create disable-client-cert-local']
+- id: converge disable-client-cert-local
+  waitFor:
+    - create disable-client-cert-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge']
-- id: verify
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge disable-client-cert-local']
+- id: verify disable-client-cert-local
+  waitFor:
+    - converge disable-client-cert-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify']
-- id: destroy
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify disable-client-cert-local']
+- id: destroy disable-client-cert-local
+  waitFor:
+    - verify disable-client-cert-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local']
+- id: create shared-vpc-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create shared-vpc-local']
+- id: converge shared-vpc-local
+  waitFor:
+    - create shared-vpc-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge shared-vpc-local']
+- id: verify shared-vpc-local
+  waitFor:
+    - converge shared-vpc-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify shared-vpc-local']
+- id: destroy shared-vpc-local
+  waitFor:
+    - verify shared-vpc-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy shared-vpc-local']
+- id: create simple-regional-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-local']
+- id: converge simple-regional-local
+  waitFor:
+    - create simple-regional-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-local']
+- id: verify simple-regional-local
+  waitFor:
+    - converge simple-regional-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-local']
+- id: destroy simple-regional-local
+  waitFor:
+    - verify simple-regional-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-local']
+- id: create simple-regional-private-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-private-local']
+- id: converge simple-regional-private-local
+  waitFor:
+    - create simple-regional-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-private-local']
+- id: verify simple-regional-private-local
+  waitFor:
+    - converge simple-regional-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-private-local']
+- id: destroy simple-regional-private-local
+  waitFor:
+    - verify simple-regional-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-private-local']
+- id: create simple-zonal-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-zonal-local']
+- id: converge simple-zonal-local
+  waitFor:
+    - create simple-zonal-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local']
+- id: verify simple-zonal-local
+  waitFor:
+    - converge simple-zonal-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local']
+- id: destroy simple-zonal-local
+  waitFor:
+    - verify simple-zonal-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local']
+- id: create simple-zonal-private-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-zonal-private-local']
+- id: converge simple-zonal-private-local
+  waitFor:
+    - create simple-zonal-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-private-local']
+- id: verify simple-zonal-private-local
+  waitFor:
+    - converge simple-zonal-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-private-local']
+- id: destroy simple-zonal-private-local
+  waitFor:
+    - verify simple-zonal-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-private-local']
+- id: create stub-domains-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create stub-domains-local']
+- id: converge stub-domains-local
+  waitFor:
+    - create stub-domains-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge stub-domains-local']
+- id: verify stub-domains-local
+  waitFor:
+    - converge stub-domains-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify stub-domains-local']
+- id: destroy stub-domains-local
+  waitFor:
+    - verify stub-domains-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy stub-domains-local']
+- id: create upstream-nameservers-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create upstream-nameservers-local']
+- id: converge upstream-nameservers-local
+  waitFor:
+    - create upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge upstream-nameservers-local']
+- id: verify upstream-nameservers-local
+  waitFor:
+    - converge upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify upstream-nameservers-local']
+- id: destroy upstream-nameservers-local
+  waitFor:
+    - verify upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy upstream-nameservers-local']
+- id: create stub-domains-upstream-nameservers-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create stub-domains-upstream-nameservers-local']
+- id: converge stub-domains-upstream-nameservers-local
+  waitFor:
+    - create stub-domains-upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge stub-domains-upstream-nameservers-local']
+- id: verify stub-domains-upstream-nameservers-local
+  waitFor:
+    - converge stub-domains-upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify stub-domains-upstream-nameservers-local']
+- id: destroy stub-domains-upstream-nameservers-local
+  waitFor:
+    - verify stub-domains-upstream-nameservers-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy stub-domains-upstream-nameservers-local']
+- id: create workload-metadata-config-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create workload-metadata-config-local']
+- id: converge workload-metadata-config-local
+  waitFor:
+    - create workload-metadata-config-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge workload-metadata-config-local']
+- id: verify workload-metadata-config-local
+  waitFor:
+    - converge workload-metadata-config-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify workload-metadata-config-local']
+- id: destroy workload-metadata-config-local
+  waitFor:
+    - verify workload-metadata-config-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local']
 tags:
 - 'ci'
 - 'integration'

From 3b83eb379e22b2718f12b8f184d4efdf8f63f83e Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Tue, 22 Oct 2019 16:06:32 -0400
Subject: [PATCH 27/90] Update Docker image to 0.4.3

---
 Makefile                   | 2 +-
 build/int.cloudbuild.yaml  | 2 +-
 build/lint.cloudbuild.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 5039822a75..b6471861d6 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.3
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index c1efa77057..cc1dd0dcfe 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -226,4 +226,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'
diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml
index 3b7306297c..d08fe185bd 100644
--- a/build/lint.cloudbuild.yaml
+++ b/build/lint.cloudbuild.yaml
@@ -24,4 +24,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'

From c197597d17f8cf266d4a8ee8e24be7a9f609c14d Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Wed, 23 Oct 2019 10:14:29 -0400
Subject: [PATCH 28/90] Regenerate modules

---
 README.md                                     | 26 +++++++++++++++++--
 .../README.md                                 |  3 +++
 .../main.tf                                   |  3 ++-
 .../networks.tf                               |  2 +-
 .../beta-private-cluster-update-variant/sa.tf |  2 +-
 .../variables.tf                              |  9 ++++++-
 modules/beta-private-cluster/README.md        |  3 +--
 modules/beta-public-cluster/README.md         |  7 +++--
 .../private-cluster-update-variant/README.md  | 22 ++++++++++++++++
 .../private-cluster-update-variant/main.tf    |  3 ++-
 .../networks.tf                               |  2 +-
 modules/private-cluster-update-variant/sa.tf  |  2 +-
 .../variables.tf                              |  9 ++++++-
 modules/private-cluster/README.md             | 22 ++++++++++++++--
 14 files changed, 99 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index b269a1772f..281cfe7c73 100644
--- a/README.md
+++ b/README.md
@@ -129,46 +129,63 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
 | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
 | master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
 | name | The name of the cluster (required) | string | n/a | yes |
 | network | The VPC network to host the cluster in (required) | string | n/a | yes |
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| region | The region to host the cluster in (required) | string | n/a | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -180,9 +197,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -194,9 +214,11 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 0fc0068e96..17eabe56d2 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -258,6 +258,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index fc38644871..63bf31ac78 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google-beta
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf
index 14ea500e03..2456654130 100644
--- a/modules/beta-private-cluster-update-variant/networks.tf
+++ b/modules/beta-private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google-beta
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/beta-private-cluster-update-variant/sa.tf
+++ b/modules/beta-private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 9a869a830f..ee2d5b5556 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -267,6 +268,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 425ef67fa1..fc35c08c54 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -188,9 +188,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| region | The region to host the cluster in (required) | string | n/a | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 73cf4ea4c5..2b2ac1e15b 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -141,10 +141,13 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
 | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
 | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -162,6 +165,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
 | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
 | master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
 | name | The name of the cluster (required) | string | n/a | yes |
 | network | The VPC network to host the cluster in (required) | string | n/a | yes |
@@ -179,9 +183,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| region | The region to host the cluster in (required) | string | n/a | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index e817361124..72491fd134 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -134,26 +134,35 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
@@ -166,17 +175,22 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
 | region | The region to host the cluster in (required) | string | n/a | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -188,9 +202,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -202,9 +219,11 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
@@ -237,6 +256,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index bfe746401c..2bd1c40d14 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf
index a382073dc0..aae034eee5 100644
--- a/modules/private-cluster-update-variant/networks.tf
+++ b/modules/private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf
index 9e063fcc22..c7f34e4fbb 100644
--- a/modules/private-cluster-update-variant/sa.tf
+++ b/modules/private-cluster-update-variant/sa.tf
@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 8008e08975..a425c13d9a 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -257,6 +258,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index a9e780d24b..791ebbbd17 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -134,26 +134,35 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
@@ -166,18 +175,22 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| region | The region to host the cluster in (required) | string | n/a | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -189,9 +202,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -203,9 +219,11 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

From 956e504f2d93ce7fa1d468737542ef58ccb84f3e Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Wed, 23 Oct 2019 10:23:15 -0400
Subject: [PATCH 29/90] Regenerate docs

---
 README.md                                     | 26 ++-----------------
 .../README.md                                 |  3 ++-
 modules/beta-private-cluster/README.md        |  3 ++-
 modules/beta-public-cluster/README.md         |  7 ++---
 .../private-cluster-update-variant/README.md  | 22 ++--------------
 modules/private-cluster/README.md             | 22 ++--------------
 6 files changed, 12 insertions(+), 71 deletions(-)

diff --git a/README.md b/README.md
index 281cfe7c73..b269a1772f 100644
--- a/README.md
+++ b/README.md
@@ -129,63 +129,46 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
 | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
 | master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
 | name | The name of the cluster (required) | string | n/a | yes |
 | network | The VPC network to host the cluster in (required) | string | n/a | yes |
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -197,12 +180,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -214,11 +194,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 17eabe56d2..d6eee7e7f7 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -188,8 +188,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index fc35c08c54..425ef67fa1 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -188,8 +188,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 2b2ac1e15b..73cf4ea4c5 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -141,13 +141,10 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
 | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
 | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -165,7 +162,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
 | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
 | master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
 | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
 | name | The name of the cluster (required) | string | n/a | yes |
 | network | The VPC network to host the cluster in (required) | string | n/a | yes |
@@ -183,8 +179,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index 72491fd134..e0ef130d3e 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -134,35 +134,26 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
@@ -175,22 +166,18 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -202,12 +189,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -219,11 +203,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 791ebbbd17..a9e780d24b 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -134,35 +134,26 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
 | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
 | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
@@ -175,22 +166,18 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy | Enable network policy addon | bool | `"false"` | no |
 | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
@@ -202,12 +189,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | Name | Description |
 |------|-------------|
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
 | location | Cluster location (region if regional cluster, zone if zonal cluster) |
 | logging\_service | Logging service used |
@@ -219,11 +203,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | network\_policy\_enabled | Whether network policy enabled |
 | node\_pools\_names | List of node pools names |
 | node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

From 86eb36db82147f18003b7120efdc0b90a39719ab Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Wed, 23 Oct 2019 11:27:42 -0400
Subject: [PATCH 30/90] Update Docker image to 0.4.5

This version fixes `check_documentation` to ignore the autogen
directory.
---
 Makefile                   | 2 +-
 build/int.cloudbuild.yaml  | 2 +-
 build/lint.cloudbuild.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index b6471861d6..52c9deaf8f 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.3
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.5
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index cc1dd0dcfe..379205db45 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -226,4 +226,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5'
diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml
index d08fe185bd..d5b2622e4f 100644
--- a/build/lint.cloudbuild.yaml
+++ b/build/lint.cloudbuild.yaml
@@ -24,4 +24,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5'

From 71449a9ba4c290c32bc14413cd3ee4bb569b2b3d Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Wed, 23 Oct 2019 11:28:07 -0400
Subject: [PATCH 31/90] Remove Inputs, Outputs tables from autogen README

This content needs to be generated specifically for each module. It
does not make sense to include it in the autogen template.
---
 autogen/README.md | 95 -----------------------------------------------
 1 file changed, 95 deletions(-)

diff --git a/autogen/README.md b/autogen/README.md
index dc0b63b003..300717fdd2 100644
--- a/autogen/README.md
+++ b/autogen/README.md
@@ -139,101 +139,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
-| region | Cluster region |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
-| zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

From 947cd1b5f527bbb491ad6f5feb7d87a8caf2cee5 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Thu, 24 Oct 2019 10:31:52 +0800
Subject: [PATCH 32/90] Regen code and docs

---
 modules/beta-private-cluster-update-variant/README.md | 1 +
 scripts/wait-for-cluster.sh                           | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index feb8d4584c..c2f9aeffd4 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -192,6 +192,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 42c9841fec..7bf9919b8f 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From cad1ad3c1f5b15dc2a3bce72a27bfbd96487f7ad Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Thu, 24 Oct 2019 12:21:57 +0800
Subject: [PATCH 33/90] Use provider version 2.18.0

---
 test/setup/versions.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/setup/versions.tf b/test/setup/versions.tf
index efbd8ea517..51f6a433b0 100644
--- a/test/setup/versions.tf
+++ b/test/setup/versions.tf
@@ -19,9 +19,9 @@ terraform {
 }
 
 provider "google" {
-  version = "~> 2.13.0"
+  version = "~> 2.18.0"
 }
 
 provider "google-beta" {
-  version = "~> 2.13.0"
+  version = "~> 2.18.0"
 }

From a219515a20934f1e4df6789d9f903afd6dc92f0a Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 24 Oct 2019 01:46:46 -0500
Subject: [PATCH 34/90] fix svc account description

---
 examples/deploy_service/README.md                     | 2 +-
 examples/deploy_service/outputs.tf                    | 2 +-
 examples/disable_client_cert/README.md                | 2 +-
 examples/disable_client_cert/outputs.tf               | 2 +-
 examples/node_pool/README.md                          | 2 +-
 examples/node_pool/outputs.tf                         | 2 +-
 examples/shared_vpc/README.md                         | 2 +-
 examples/shared_vpc/outputs.tf                        | 2 +-
 examples/simple_regional/README.md                    | 2 +-
 examples/simple_regional/outputs.tf                   | 2 +-
 examples/simple_regional_beta/outputs.tf              | 2 +-
 examples/simple_regional_private/README.md            | 2 +-
 examples/simple_regional_private/outputs.tf           | 2 +-
 examples/simple_regional_private_beta/outputs.tf      | 2 +-
 examples/simple_zonal/README.md                       | 2 +-
 examples/simple_zonal/outputs.tf                      | 2 +-
 examples/simple_zonal_private/README.md               | 2 +-
 examples/simple_zonal_private/outputs.tf              | 2 +-
 examples/stub_domains/README.md                       | 2 +-
 examples/stub_domains/outputs.tf                      | 2 +-
 examples/stub_domains_private/README.md               | 2 +-
 examples/stub_domains_private/outputs.tf              | 2 +-
 examples/stub_domains_upstream_nameservers/outputs.tf | 2 +-
 examples/upstream_nameservers/outputs.tf              | 2 +-
 examples/workload_metadata_config/outputs.tf          | 2 +-
 25 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md
index 5dcb7ca7a7..e13981c450 100644
--- a/examples/deploy_service/README.md
+++ b/examples/deploy_service/README.md
@@ -37,7 +37,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/deploy_service/outputs.tf b/examples/deploy_service/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/deploy_service/outputs.tf
+++ b/examples/deploy_service/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/disable_client_cert/README.md b/examples/disable_client_cert/README.md
index 14dd6545c0..2f531b9906 100644
--- a/examples/disable_client_cert/README.md
+++ b/examples/disable_client_cert/README.md
@@ -36,7 +36,7 @@ This example illustrates how to create a simple cluster and disable deprecated s
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/disable_client_cert/outputs.tf b/examples/disable_client_cert/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/disable_client_cert/outputs.tf
+++ b/examples/disable_client_cert/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md
index 9215f091cb..237b3f0b6f 100644
--- a/examples/node_pool/README.md
+++ b/examples/node_pool/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a cluster with multiple custom node-pool
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/node_pool/outputs.tf b/examples/node_pool/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/node_pool/outputs.tf
+++ b/examples/node_pool/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md
index 3b0f5a6157..964a346349 100644
--- a/examples/shared_vpc/README.md
+++ b/examples/shared_vpc/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a simple cluster where the host network i
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/shared_vpc/outputs.tf b/examples/shared_vpc/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/shared_vpc/outputs.tf
+++ b/examples/shared_vpc/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md
index fb209e47b5..d1d986cd6e 100644
--- a/examples/simple_regional/README.md
+++ b/examples/simple_regional/README.md
@@ -31,7 +31,7 @@ This example illustrates how to create a simple cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_regional/outputs.tf b/examples/simple_regional/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional/outputs.tf
+++ b/examples/simple_regional/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional_beta/outputs.tf b/examples/simple_regional_beta/outputs.tf
index ad152e186c..0d770aa809 100644
--- a/examples/simple_regional_beta/outputs.tf
+++ b/examples/simple_regional_beta/outputs.tf
@@ -29,6 +29,6 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md
index 8175482731..917c097951 100644
--- a/examples/simple_regional_private/README.md
+++ b/examples/simple_regional_private/README.md
@@ -31,7 +31,7 @@ This example illustrates how to create a simple private cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional_private/outputs.tf
+++ b/examples/simple_regional_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional_private_beta/outputs.tf b/examples/simple_regional_private_beta/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional_private_beta/outputs.tf
+++ b/examples/simple_regional_private_beta/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md
index 691f95c719..c086ea3a4b 100644
--- a/examples/simple_zonal/README.md
+++ b/examples/simple_zonal/README.md
@@ -31,7 +31,7 @@ This example illustrates how to create a simple cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_zonal/outputs.tf
+++ b/examples/simple_zonal/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md
index e576800d72..83cb7c575b 100644
--- a/examples/simple_zonal_private/README.md
+++ b/examples/simple_zonal_private/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a simple private cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_zonal_private/outputs.tf
+++ b/examples/simple_zonal_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md
index 126a1cd54c..bc4491b880 100644
--- a/examples/stub_domains/README.md
+++ b/examples/stub_domains/README.md
@@ -36,7 +36,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/stub_domains/outputs.tf b/examples/stub_domains/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains/outputs.tf
+++ b/examples/stub_domains/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains_private/README.md b/examples/stub_domains_private/README.md
index ee4b89fa7f..205d5fdf76 100644
--- a/examples/stub_domains_private/README.md
+++ b/examples/stub_domains_private/README.md
@@ -38,7 +38,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/stub_domains_private/outputs.tf b/examples/stub_domains_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains_private/outputs.tf
+++ b/examples/stub_domains_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains_upstream_nameservers/outputs.tf b/examples/stub_domains_upstream_nameservers/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains_upstream_nameservers/outputs.tf
+++ b/examples/stub_domains_upstream_nameservers/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/upstream_nameservers/outputs.tf b/examples/upstream_nameservers/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/upstream_nameservers/outputs.tf
+++ b/examples/upstream_nameservers/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/workload_metadata_config/outputs.tf b/examples/workload_metadata_config/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/workload_metadata_config/outputs.tf
+++ b/examples/workload_metadata_config/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 

From 2c3eb0c1baf2b9321b8e0352804843a11cae81cb Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Thu, 24 Oct 2019 18:45:08 +0300
Subject: [PATCH 35/90] Fix destruction order

 * Add dependency on k8s endpoint
 * Fix lint-tests
---
 autogen/README.md                             |  2 +-
 examples/simple_zonal/acm.tf                  | 15 ++++++------
 examples/simple_zonal/outputs.tf              |  3 ++-
 examples/simple_zonal/variables.tf            |  3 ++-
 modules/acm/README.md                         | 12 ++++++----
 modules/acm/main.tf                           | 21 ++++++++++++++--
 modules/acm/outputs.tf                        | 19 ++++++++++++++-
 modules/acm/templates/acm-config.yml.tpl      |  3 ++-
 modules/acm/variables.tf                      | 24 ++++++++++++++++++-
 .../README.md                                 |  2 +-
 .../main.tf                                   |  3 ++-
 .../networks.tf                               |  2 +-
 .../variables.tf                              |  3 ++-
 .../private-cluster-update-variant/README.md  |  2 +-
 .../private-cluster-update-variant/main.tf    |  3 ++-
 .../networks.tf                               |  2 +-
 .../variables.tf                              |  3 ++-
 17 files changed, 94 insertions(+), 28 deletions(-)

diff --git a/autogen/README.md b/autogen/README.md
index 421e4a2605..14d7409c7f 100644
--- a/autogen/README.md
+++ b/autogen/README.md
@@ -195,7 +195,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal/acm.tf
index 1c38d22c99..47fdce1d86 100644
--- a/examples/simple_zonal/acm.tf
+++ b/examples/simple_zonal/acm.tf
@@ -15,11 +15,12 @@
  */
 
 module "acm" {
-  source       = "../../modules/acm"
-  project_id   = var.project_id
-  location     = module.gke.location
-  cluster_name = module.gke.name
-  sync_repo    = var.acm_sync_repo
-  sync_branch  = var.acm_sync_branch
-  policy_dir   = var.acm_policy_dir
+  source           = "../../modules/acm"
+  project_id       = var.project_id
+  location         = module.gke.location
+  cluster_name     = module.gke.name
+  sync_repo        = var.acm_sync_repo
+  sync_branch      = var.acm_sync_branch
+  policy_dir       = var.acm_policy_dir
+  cluster_endpoint = module.gke.endpoint
 }
diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf
index 143710f8b2..2492208f7c 100644
--- a/examples/simple_zonal/outputs.tf
+++ b/examples/simple_zonal/outputs.tf
@@ -36,4 +36,5 @@ output "service_account" {
 output "acm_git_creds_public" {
   description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository."
   value       = module.acm.git_creds_public
-}
\ No newline at end of file
+}
+
diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal/variables.tf
index adc92dba17..39a97b02d0 100644
--- a/examples/simple_zonal/variables.tf
+++ b/examples/simple_zonal/variables.tf
@@ -64,4 +64,5 @@ variable "acm_policy_dir" {
   description = "Subfolder containing configs in Ahtons config management Git repo"
   type        = string
   default     = "foo-corp"
-}
\ No newline at end of file
+}
+
diff --git a/modules/acm/README.md b/modules/acm/README.md
index 99a40bc840..40fdb939db 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -1,6 +1,6 @@
-# Terraform Kubernetes Engine ACM Submodule 
+# Terraform Kubernetes Engine ACM Submodule
 
-This module installs Anthos Config Management (ACM) in a Kubernetes cluster. 
+This module installs Anthos Config Management (ACM) in a Kubernetes cluster.
 To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/).
 
 
@@ -9,11 +9,11 @@ To find out more about ACM check [documentation](https://cloud.google.com/anthos
 In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md)
 plus **roles/container.admin** role.
 
-## Usage example 
+## Usage example
 
 See [examples/simple_zonal](../../examples/simple_zonal) cluster example.
 
-## Installation 
+## Installation
 
 Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page.
 Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public**
@@ -25,6 +25,7 @@ output variable to your account at your Git server.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes |
 | cluster\_name | The name of the cluster. | string | n/a | yes |
 | location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes |
 | policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes |
@@ -38,4 +39,5 @@ output variable to your account at your Git server.
 |------|-------------|
 | git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. |
 
- <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
\ No newline at end of file
+ <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 7a2a6019d4..10e55e9f88 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -1,5 +1,21 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 locals {
-  cluster_endpoint       = "https://${data.google_container_cluster.primary.endpoint}"
+  cluster_endpoint       = "https://${var.cluster_endpoint}"
   token                  = data.google_client_config.default.access_token
   cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate
 }
@@ -87,4 +103,5 @@ resource "null_resource" "acm_config" {
     null_resource.acm_operator,
     null_resource.git_creds_secret,
   ]
-}
\ No newline at end of file
+}
+
diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf
index b37d521c0e..0ebca72e42 100644
--- a/modules/acm/outputs.tf
+++ b/modules/acm/outputs.tf
@@ -1,4 +1,21 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 output "git_creds_public" {
   description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository."
   value       = tls_private_key.git_creds.public_key_openssh
-}
\ No newline at end of file
+}
+
diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl
index af60b8690a..8781f9db79 100644
--- a/modules/acm/templates/acm-config.yml.tpl
+++ b/modules/acm/templates/acm-config.yml.tpl
@@ -9,4 +9,5 @@ spec:
     syncRepo: ${sync_repo}
     syncBranch: ${sync_branch}
     secretType: ${secret_type}
-    policyDir: ${policy_dir}
\ No newline at end of file
+    policyDir: ${policy_dir}
+
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
index 9640135b24..94b720e479 100644
--- a/modules/acm/variables.tf
+++ b/modules/acm/variables.tf
@@ -1,3 +1,19 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 variable "cluster_name" {
   description = "The name of the cluster."
   type        = string
@@ -27,4 +43,10 @@ variable "sync_branch" {
 variable "policy_dir" {
   description = "Subfolder containing configs in Ahtons config management Git repo"
   type        = string
-}
\ No newline at end of file
+}
+
+variable "cluster_endpoint" {
+  description = "Kubernetes cluster endpoint."
+  type        = string
+}
+
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 0fc0068e96..41279cc522 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -188,7 +188,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index fc38644871..63bf31ac78 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google-beta
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf
index 14ea500e03..2456654130 100644
--- a/modules/beta-private-cluster-update-variant/networks.tf
+++ b/modules/beta-private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google-beta
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 9a869a830f..d783248ea2 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index e817361124..8b0a140c4c 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -174,7 +174,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index bfe746401c..2bd1c40d14 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version
diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf
index a382073dc0..aae034eee5 100644
--- a/modules/private-cluster-update-variant/networks.tf
+++ b/modules/private-cluster-update-variant/networks.tf
@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 8008e08975..28b744d868 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {

From 98b0690c07883b513dd944b51abf6a7e8459caf6 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 24 Oct 2019 16:29:45 -0400
Subject: [PATCH 36/90] Add #258

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c78c043f18..d39fb93396 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 ### Added
 
+* Added ability to skip local-exec provisioners. [#258]
 * Added [private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/private-cluster-update-variant) and [beta private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant) variants which allow node pools to be created before being destroyed. [#256]
 * Add a parameter `registry_project_id` to allow connecting to registries in other projects. [#273]
 
@@ -205,6 +206,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
+[#258]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/258
 [#273]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/273
 [#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247
 [#256]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/256

From 2438e71a68f118026dd7bcd751a73e31feea30a6 Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Thu, 24 Oct 2019 16:52:25 -0400
Subject: [PATCH 37/90] Add entry for 5.1.0 to CHANGELOG

Add entries:
- #282
- #284
- #285
---
 CHANGELOG.md | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d39fb93396..5e609c0eca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,9 +8,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 ## [Unreleased]
 
-### Changed
-
-* Made `region` variable optional for zonal clusters [#247]
+## [v5.1.0] - 2019-10-24
 
 ### Added
 
@@ -18,6 +16,15 @@ Extending the adopted spec, each change should have a link to its corresponding
 * Added [private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/private-cluster-update-variant) and [beta private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant) variants which allow node pools to be created before being destroyed. [#256]
 * Add a parameter `registry_project_id` to allow connecting to registries in other projects. [#273]
 
+### Changed
+
+* Made `region` variable optional for zonal clusters. [#247]
+* Made default metadata, labels, and tags optional. [#282]
+
+### Fixed
+
+* Authenticate gcloud in wait-for-cluster.sh using value of `GOOGLE_APPLICATION_CREDENTIALS`. [#284] [#285]
+
 ## [v5.0.0] - 2019-09-25
 v5.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v5.0.md).
 
@@ -191,7 +198,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 * Initial release of module.
 
-[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...HEAD
+[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...HEAD
+[v5.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...v5.1.0
 [v5.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.1.0...v5.0.0
 [v4.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.0.0...v4.1.0
 [v4.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v3.0.0...v4.0.0
@@ -206,11 +214,14 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
-[#258]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/258
+[#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285
+[#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284
+[#282]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/282
 [#273]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/273
-[#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247
+[#258]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/258
 [#256]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/256
 [#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248
+[#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247
 [#228]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/228
 [#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238
 [#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241

From ed765c478b01d763b422a1e53a7ebec0034971ab Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 24 Oct 2019 17:51:57 -0500
Subject: [PATCH 38/90] minor nits, clean fixtures

---
 .../simple_regional_with_networking/README.md | 14 ++---
 .../simple_regional_with_networking/main.tf   | 10 ++--
 .../outputs.tf                                | 14 ++++-
 .../test_outputs.tf                           | 59 ++++++++++++++++++-
 .../variables.tf                              | 10 ++--
 .../example.tf                                |  9 +--
 .../outputs.tf                                |  6 +-
 .../variables.tf                              | 20 -------
 8 files changed, 91 insertions(+), 51 deletions(-)
 mode change 120000 => 100644 examples/simple_regional_with_networking/test_outputs.tf

diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index e24ee546fb..2f9d844a61 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -7,9 +7,9 @@ This example illustrates how to create a VPC and a simple cluster.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
-| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no |
-| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no |
+| cluster\_name | The name for the GKE cluster | string | `"gke-on-vpc-cluster"` | no |
+| ip\_range\_pods\_name | The secondary ip range to use for pods | string | `"ip-range-pods"` | no |
+| ip\_range\_services\_name | The secondary ip range to use for pods | string | `"ip-range-scv"` | no |
 | network | The VPC network created to host the cluster in | string | `"gke-network"` | no |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | `"us-central1"` | no |
@@ -22,8 +22,8 @@ This example illustrates how to create a VPC and a simple cluster.
 | ca\_certificate |  |
 | client\_token |  |
 | cluster\_name | Cluster name |
-| ip\_range\_pods | The secondary IP range used for pods |
-| ip\_range\_services | The secondary IP range used for services |
+| ip\_range\_pods\_name | The secondary IP range used for pods |
+| ip\_range\_services\_name | The secondary IP range used for services |
 | kubernetes\_endpoint |  |
 | location |  |
 | master\_kubernetes\_version | The master Kubernetes version |
@@ -31,8 +31,8 @@ This example illustrates how to create a VPC and a simple cluster.
 | network\_name | The name of the VPC being created |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| subnet\_names | The name of the subnet being created |
+| service\_account | The default service account used for running nodes. |
+| subnet\_name | The name of the subnet being created |
 | subnet\_secondary\_ranges | The secondary ranges associated with the subnet |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index fc53436f2d..7789d73cdd 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -36,11 +36,11 @@ module "gcp-network" {
   secondary_ranges = {
     "${var.subnetwork}" = [
       {
-        range_name    = var.ip_range_pods
+        range_name    = var.ip_range_pods_name
         ip_cidr_range = "192.168.0.0/18"
       },
       {
-        range_name    = var.ip_range_services
+        range_name    = var.ip_range_services_name
         ip_cidr_range = "192.168.64.0/18"
       },
     ]
@@ -50,13 +50,13 @@ module "gcp-network" {
 module "gke" {
   source                 = "../../"
   project_id             = var.project_id
-  name                   = "simple-regional-cluster-${var.cluster_name_suffix}"
+  name                   = var.cluster_name
   regional               = true
   region                 = var.region
   network                = module.gcp-network.network_name
   subnetwork             = module.gcp-network.subnets_names[0]
-  ip_range_pods          = var.ip_range_pods
-  ip_range_services      = var.ip_range_services
+  ip_range_pods          = var.ip_range_pods_name
+  ip_range_services      = var.ip_range_services_name
   create_service_account = true
 }
 
diff --git a/examples/simple_regional_with_networking/outputs.tf b/examples/simple_regional_with_networking/outputs.tf
index 6cf2ab5eab..4bd320aae8 100644
--- a/examples/simple_regional_with_networking/outputs.tf
+++ b/examples/simple_regional_with_networking/outputs.tf
@@ -29,21 +29,29 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
 output "network_name" {
   description = "The name of the VPC being created"
   value       = module.gcp-network.network_name
 }
 
-output "subnet_names" {
+output "subnet_name" {
   description = "The name of the subnet being created"
-  value       = module.gcp-network.subnets_names
+  value       = module.gcp-network.subnets_name
 }
 
 output "subnet_secondary_ranges" {
   description = "The secondary ranges associated with the subnet"
   value       = module.gcp-network.subnets_secondary_ranges
 }
+
+
+
diff --git a/examples/simple_regional_with_networking/test_outputs.tf b/examples/simple_regional_with_networking/test_outputs.tf
deleted file mode 120000
index 17b34213ba..0000000000
--- a/examples/simple_regional_with_networking/test_outputs.tf
+++ /dev/null
@@ -1 +0,0 @@
-../../test/fixtures/all_examples/test_outputs.tf
\ No newline at end of file
diff --git a/examples/simple_regional_with_networking/test_outputs.tf b/examples/simple_regional_with_networking/test_outputs.tf
new file mode 100644
index 0000000000..a703679105
--- /dev/null
+++ b/examples/simple_regional_with_networking/test_outputs.tf
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// These outputs are used to test the module with kitchen-terraform
+// They do not need to be included in real-world uses of this module
+
+output "project_id" {
+  value = var.project_id
+}
+
+output "region" {
+  value = module.gke.region
+}
+
+output "network" {
+  value = var.network
+}
+
+output "subnetwork" {
+  value = var.subnetwork
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "ip_range_pods_name" {
+  description = "The secondary IP range used for pods"
+  value       = var.ip_range_pods_name
+}
+
+output "ip_range_services_name" {
+  description = "The secondary IP range used for services"
+  value       = var.ip_range_services_name
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "master_kubernetes_version" {
+  description = "The master Kubernetes version"
+  value       = module.gke.master_version
+}
diff --git a/examples/simple_regional_with_networking/variables.tf b/examples/simple_regional_with_networking/variables.tf
index e9045a5c0d..8e9c0688de 100644
--- a/examples/simple_regional_with_networking/variables.tf
+++ b/examples/simple_regional_with_networking/variables.tf
@@ -18,9 +18,9 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "cluster_name_suffix" {
-  description = "A suffix to append to the default cluster name"
-  default     = ""
+variable "cluster_name" {
+  description = "The name for the GKE cluster"
+  default     = "gke-on-vpc-cluster"
 }
 
 variable "region" {
@@ -38,12 +38,12 @@ variable "subnetwork" {
   default     = "gke-subnet"
 }
 
-variable "ip_range_pods" {
+variable "ip_range_pods_name" {
   description = "The secondary ip range to use for pods"
   default     = "ip-range-pods"
 }
 
-variable "ip_range_services" {
+variable "ip_range_services_name" {
   description = "The secondary ip range to use for pods"
   default     = "ip-range-scv"
 }
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index 59f6c27ba2..ace90efb3d 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -23,11 +23,6 @@ resource "random_string" "suffix" {
 module "example" {
   source = "../../../examples/simple_regional_with_networking"
 
-  project_id          = var.project_id
-  cluster_name_suffix = "-${random_string.suffix.result}"
-  region              = var.region
-  network             = "${var.network}-${random_string.suffix.result}"
-  subnetwork          = "${var.subnetwork}-${random_string.suffix.result}"
-  ip_range_pods       = "${var.ip_range_pods}-${random_string.suffix.result}"
-  ip_range_services   = "${var.ip_range_services}-${random_string.suffix.result}"
+  project_id = var.project_id
+  region     = var.region
 }
diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf
index 2a13237983..08f9a8a2e8 100644
--- a/test/fixtures/simple_regional_with_networking/outputs.tf
+++ b/test/fixtures/simple_regional_with_networking/outputs.tf
@@ -43,7 +43,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.example.service_account
 }
 
@@ -64,10 +64,10 @@ output "region" {
 
 output "ip_range_pods_name" {
   description = "The secondary range name for pods"
-  value       = module.example.ip_range_pods
+  value       = module.example.ip_range_pods_name
 }
 
 output "ip_range_services_name" {
   description = "The secondary range name for services"
-  value       = module.example.ip_range_services
+  value       = module.example.ip_range_services_name
 }
diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf
index 989451fe32..e9310a56c5 100644
--- a/test/fixtures/simple_regional_with_networking/variables.tf
+++ b/test/fixtures/simple_regional_with_networking/variables.tf
@@ -22,23 +22,3 @@ variable "region" {
   description = "The region to host the cluster in"
   default     = "us-east4"
 }
-
-variable "network" {
-  description = "The VPC network created to host the cluster in"
-  default     = "gke-network"
-}
-
-variable "subnetwork" {
-  description = "The subnetwork created to host the cluster in"
-  default     = "gke-subnet"
-}
-
-variable "ip_range_pods" {
-  description = "The secondary ip range to use for pods"
-  default     = "ip-range-pods"
-}
-
-variable "ip_range_services" {
-  description = "The secondary ip range to use for pods"
-  default     = "ip-range-scv"
-}

From 7e21e69017d01a2c48be84d33090f34323cb7bf4 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 24 Oct 2019 19:40:12 -0500
Subject: [PATCH 39/90] typo

---
 examples/simple_regional_with_networking/outputs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/simple_regional_with_networking/outputs.tf b/examples/simple_regional_with_networking/outputs.tf
index 4bd320aae8..a26f0275e0 100644
--- a/examples/simple_regional_with_networking/outputs.tf
+++ b/examples/simple_regional_with_networking/outputs.tf
@@ -45,7 +45,7 @@ output "network_name" {
 
 output "subnet_name" {
   description = "The name of the subnet being created"
-  value       = module.gcp-network.subnets_name
+  value       = module.gcp-network.subnets_names
 }
 
 output "subnet_secondary_ranges" {

From ef865736b878a0d9a595645b09831858de7ce8b6 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 24 Oct 2019 01:46:46 -0500
Subject: [PATCH 40/90] fix svc account description

---
 examples/deploy_service/README.md                     | 2 +-
 examples/deploy_service/outputs.tf                    | 2 +-
 examples/disable_client_cert/README.md                | 2 +-
 examples/disable_client_cert/outputs.tf               | 2 +-
 examples/node_pool/README.md                          | 2 +-
 examples/node_pool/outputs.tf                         | 2 +-
 examples/shared_vpc/README.md                         | 2 +-
 examples/shared_vpc/outputs.tf                        | 2 +-
 examples/simple_regional/README.md                    | 2 +-
 examples/simple_regional/outputs.tf                   | 2 +-
 examples/simple_regional_beta/outputs.tf              | 2 +-
 examples/simple_regional_private/README.md            | 2 +-
 examples/simple_regional_private/outputs.tf           | 2 +-
 examples/simple_regional_private_beta/outputs.tf      | 2 +-
 examples/simple_zonal/README.md                       | 2 +-
 examples/simple_zonal/outputs.tf                      | 2 +-
 examples/simple_zonal_private/README.md               | 2 +-
 examples/simple_zonal_private/outputs.tf              | 2 +-
 examples/stub_domains/README.md                       | 2 +-
 examples/stub_domains/outputs.tf                      | 2 +-
 examples/stub_domains_private/README.md               | 2 +-
 examples/stub_domains_private/outputs.tf              | 2 +-
 examples/stub_domains_upstream_nameservers/outputs.tf | 2 +-
 examples/upstream_nameservers/outputs.tf              | 2 +-
 examples/workload_metadata_config/outputs.tf          | 2 +-
 25 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md
index 5dcb7ca7a7..e13981c450 100644
--- a/examples/deploy_service/README.md
+++ b/examples/deploy_service/README.md
@@ -37,7 +37,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/deploy_service/outputs.tf b/examples/deploy_service/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/deploy_service/outputs.tf
+++ b/examples/deploy_service/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/disable_client_cert/README.md b/examples/disable_client_cert/README.md
index 14dd6545c0..2f531b9906 100644
--- a/examples/disable_client_cert/README.md
+++ b/examples/disable_client_cert/README.md
@@ -36,7 +36,7 @@ This example illustrates how to create a simple cluster and disable deprecated s
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/disable_client_cert/outputs.tf b/examples/disable_client_cert/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/disable_client_cert/outputs.tf
+++ b/examples/disable_client_cert/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md
index 9215f091cb..237b3f0b6f 100644
--- a/examples/node_pool/README.md
+++ b/examples/node_pool/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a cluster with multiple custom node-pool
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/node_pool/outputs.tf b/examples/node_pool/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/node_pool/outputs.tf
+++ b/examples/node_pool/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md
index 3b0f5a6157..964a346349 100644
--- a/examples/shared_vpc/README.md
+++ b/examples/shared_vpc/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a simple cluster where the host network i
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/shared_vpc/outputs.tf b/examples/shared_vpc/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/shared_vpc/outputs.tf
+++ b/examples/shared_vpc/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md
index 1f0a187b6b..4950b21e9a 100644
--- a/examples/simple_regional/README.md
+++ b/examples/simple_regional/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a simple cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_regional/outputs.tf b/examples/simple_regional/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional/outputs.tf
+++ b/examples/simple_regional/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional_beta/outputs.tf b/examples/simple_regional_beta/outputs.tf
index ad152e186c..0d770aa809 100644
--- a/examples/simple_regional_beta/outputs.tf
+++ b/examples/simple_regional_beta/outputs.tf
@@ -29,6 +29,6 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md
index 8175482731..917c097951 100644
--- a/examples/simple_regional_private/README.md
+++ b/examples/simple_regional_private/README.md
@@ -31,7 +31,7 @@ This example illustrates how to create a simple private cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional_private/outputs.tf
+++ b/examples/simple_regional_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_regional_private_beta/outputs.tf b/examples/simple_regional_private_beta/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_regional_private_beta/outputs.tf
+++ b/examples/simple_regional_private_beta/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md
index 691f95c719..c086ea3a4b 100644
--- a/examples/simple_zonal/README.md
+++ b/examples/simple_zonal/README.md
@@ -31,7 +31,7 @@ This example illustrates how to create a simple cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_zonal/outputs.tf
+++ b/examples/simple_zonal/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md
index e576800d72..83cb7c575b 100644
--- a/examples/simple_zonal_private/README.md
+++ b/examples/simple_zonal_private/README.md
@@ -32,7 +32,7 @@ This example illustrates how to create a simple private cluster.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/simple_zonal_private/outputs.tf
+++ b/examples/simple_zonal_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md
index 126a1cd54c..bc4491b880 100644
--- a/examples/stub_domains/README.md
+++ b/examples/stub_domains/README.md
@@ -36,7 +36,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/stub_domains/outputs.tf b/examples/stub_domains/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains/outputs.tf
+++ b/examples/stub_domains/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains_private/README.md b/examples/stub_domains_private/README.md
index ee4b89fa7f..205d5fdf76 100644
--- a/examples/stub_domains_private/README.md
+++ b/examples/stub_domains_private/README.md
@@ -38,7 +38,7 @@ It will:
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 
diff --git a/examples/stub_domains_private/outputs.tf b/examples/stub_domains_private/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains_private/outputs.tf
+++ b/examples/stub_domains_private/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/stub_domains_upstream_nameservers/outputs.tf b/examples/stub_domains_upstream_nameservers/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/stub_domains_upstream_nameservers/outputs.tf
+++ b/examples/stub_domains_upstream_nameservers/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/upstream_nameservers/outputs.tf b/examples/upstream_nameservers/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/upstream_nameservers/outputs.tf
+++ b/examples/upstream_nameservers/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 
diff --git a/examples/workload_metadata_config/outputs.tf b/examples/workload_metadata_config/outputs.tf
index 0d972dcd88..01a13147c2 100644
--- a/examples/workload_metadata_config/outputs.tf
+++ b/examples/workload_metadata_config/outputs.tf
@@ -29,7 +29,7 @@ output "ca_certificate" {
 }
 
 output "service_account" {
-  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  description = "The default service account used for running nodes."
   value       = module.gke.service_account
 }
 

From 934fd12c252f2ea72587b116ec22de56a4f586e1 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Fri, 25 Oct 2019 11:42:19 +0800
Subject: [PATCH 41/90] Fixes to ensure tests can be ran locally

---
 examples/node_pool/main.tf                     | 2 +-
 examples/node_pool_update_variant_beta/main.tf | 2 +-
 examples/simple_regional_beta/main.tf          | 3 +--
 examples/simple_regional_private_beta/main.tf  | 3 +--
 examples/workload_metadata_config/main.tf      | 2 +-
 test/setup/main.tf                             | 2 ++
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index 6662bb84ac..c7a7f852ae 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/node_pool_update_variant_beta/main.tf b/examples/node_pool_update_variant_beta/main.tf
index 373fd59f30..37b595f793 100644
--- a/examples/node_pool_update_variant_beta/main.tf
+++ b/examples/node_pool_update_variant_beta/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
+  version     = "~> 2.18.0"
   credentials = file(var.credentials_path)
   region      = var.region
 }
diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf
index fc95090ede..5eea5e23c7 100644
--- a/examples/simple_regional_beta/main.tf
+++ b/examples/simple_regional_beta/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
+  version     = "~> 2.18.0"
   credentials = file(var.credentials_path)
   region      = var.region
 }
@@ -42,4 +42,3 @@ module "gke" {
 
 data "google_client_config" "default" {
 }
-
diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf
index 0ca1873d86..db6c8a8204 100644
--- a/examples/simple_regional_private_beta/main.tf
+++ b/examples/simple_regional_private_beta/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
+  version     = "~> 2.18.0"
   credentials = file(var.credentials_path)
   region      = var.region
 }
@@ -62,4 +62,3 @@ module "gke" {
 
 data "google_client_config" "default" {
 }
-
diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf
index f9fb25da5b..3d2254c2da 100644
--- a/examples/workload_metadata_config/main.tf
+++ b/examples/workload_metadata_config/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/test/setup/main.tf b/test/setup/main.tf
index f974c7408e..70e10c46a3 100644
--- a/test/setup/main.tf
+++ b/test/setup/main.tf
@@ -24,6 +24,8 @@ module "gke-project" {
   folder_id         = var.folder_id
   billing_account   = var.billing_account
 
+  auto_create_network = true
+
   activate_apis = [
     "bigquery-json.googleapis.com",
     "cloudkms.googleapis.com",

From 8d8451c7112f483a2fc40424f48820688d7c289b Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Fri, 25 Oct 2019 11:29:49 +0300
Subject: [PATCH 42/90] Added sandbox-enabled-local to int tests on CI

 * Fixed lint
---
 build/int.cloudbuild.yaml               | 20 ++++++++++++++++++++
 examples/simple_regional_beta/README.md |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 379205db45..a4ce07ab4a 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -221,6 +221,26 @@ steps:
     - verify workload-metadata-config-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local']
+- id: create sandbox-enabled-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create sandbox-enabled-local']
+- id: converge sandbox-enabled-local
+  waitFor:
+    - create sandbox-enabled-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge sandbox-enabled-local']
+- id: verify sandbox-enabled-local
+  waitFor:
+    - converge sandbox-enabled-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify sandbox-enabled-local']
+- id: destroy sandbox-enabled-local
+  waitFor:
+    - verify sandbox-enabled-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy sandbox-enabled-local']
 tags:
 - 'ci'
 - 'integration'
diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md
index 02d0dba224..72bb221d9f 100644
--- a/examples/simple_regional_beta/README.md
+++ b/examples/simple_regional_beta/README.md
@@ -37,7 +37,7 @@ This example illustrates how to create a simple cluster with beta features.
 | network |  |
 | project\_id |  |
 | region |  |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| service\_account | The default service account used for running nodes. |
 | subnetwork |  |
 | zones | List of zones in which the cluster resides |
 

From 3906c7b5aefc0312c7cf25f33f91dd2da0d18ba7 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Thu, 3 Oct 2019 14:59:57 +0300
Subject: [PATCH 43/90] Update test to run all

 * Fixed #274
---
 .dockerignore                                 | 11 -----
 .kitchen.yml                                  | 45 +++++++++++--------
 examples/disable_client_cert/README.md        |  1 -
 examples/disable_client_cert/variables.tf     |  4 --
 examples/simple_regional_beta/main.tf         |  5 +--
 examples/simple_regional_beta/test_outputs.tf |  4 --
 examples/simple_regional_beta/variables.tf    |  4 --
 examples/simple_regional_private_beta/main.tf |  5 +--
 .../test_outputs.tf                           |  4 --
 .../simple_regional_private_beta/variables.tf |  4 --
 examples/simple_zonal_private/main.tf         |  2 +-
 scripts/wait-for-cluster.sh                   |  1 +
 test/ci/deploy-service.yml                    | 18 --------
 test/ci/node-pool.yml                         | 18 --------
 test/ci/shared-vpc.yml                        | 18 --------
 test/ci/simple-regional-private.yml           | 18 --------
 test/ci/simple-regional.yml                   | 18 --------
 test/ci/simple-zonal-private.yml              | 18 --------
 test/ci/simple-zonal.yml                      | 18 --------
 test/ci/stub-domains-upstream-nameservers.yml | 18 --------
 test/ci/stub-domains.yml                      | 18 --------
 test/ci/upstream-nameservers.yml              | 18 --------
 test/ci/workload-metadata-config.yml          | 19 --------
 test/fixtures/disable_client_cert/example.tf  |  1 -
 .../disable_client_cert/inspec.yml            |  3 --
 test/integration/simple_zonal/controls/gcp.rb |  2 +-
 test/integration/simple_zonal/inspec.yml      |  5 +--
 .../simple_zonal_private/inspec.yml           |  2 +-
 test/task_helper_functions.sh                 | 13 ------
 29 files changed, 36 insertions(+), 279 deletions(-)
 delete mode 100644 .dockerignore
 delete mode 100644 test/ci/deploy-service.yml
 delete mode 100644 test/ci/node-pool.yml
 delete mode 100644 test/ci/shared-vpc.yml
 delete mode 100644 test/ci/simple-regional-private.yml
 delete mode 100644 test/ci/simple-regional.yml
 delete mode 100644 test/ci/simple-zonal-private.yml
 delete mode 100644 test/ci/simple-zonal.yml
 delete mode 100644 test/ci/stub-domains-upstream-nameservers.yml
 delete mode 100644 test/ci/stub-domains.yml
 delete mode 100644 test/ci/upstream-nameservers.yml
 delete mode 100644 test/ci/workload-metadata-config.yml

diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index ab75c635fe..0000000000
--- a/.dockerignore
+++ /dev/null
@@ -1,11 +0,0 @@
-.git
-.terraform
-.terraform.d
-.kitchen
-terraform.tfstate.d
-test/fixtures/*/.terraform
-test/fixtures/*/terraform.tfstate.d
-examples/.kitchen
-examples/*/.terraform
-examples/*/terraform.tfstate.d
-
diff --git a/.kitchen.yml b/.kitchen.yml
index 9f5df5a03e..0f552f341f 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -29,15 +29,6 @@ platforms:
   - name: local
 
 suites:
-# Disabled due to issue #274
-# (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/274)
-#  - name: "deploy_service"
-#    driver:
-#      root_module_directory: test/fixtures/deploy_service
-#    verifier:
-#      systems:
-#        - name: deploy_service
-#          backend: local
   - name: "disable_client_cert"
     driver:
       root_module_directory: test/fixtures/disable_client_cert
@@ -45,15 +36,8 @@ suites:
       systems:
         - name: disable_client_cert
           backend: local
-# Disabled due to issue #274
-# (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/274)
-#  - name: "node_pool"
-#    driver:
-#      root_module_directory: test/fixtures/node_pool
-#    verifier:
-#      systems:
-#        - name: node_pool
-#          backend: local
+          controls:
+            - gcloud
   - name: "shared_vpc"
     driver:
       root_module_directory: test/fixtures/shared_vpc
@@ -95,6 +79,8 @@ suites:
       systems:
         - name: simple_zonal_private
           backend: local
+          controls:
+            - gcloud
   - name: "stub_domains"
     driver:
       root_module_directory: test/fixtures/stub_domains
@@ -102,6 +88,9 @@ suites:
       systems:
         - name: stub_domains
           backend: local
+          controls:
+            - gcloud
+            - kubectl
 # Disabled due to issue #264
 # (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/264)
 #  - name: stub_domains_private
@@ -131,3 +120,23 @@ suites:
       systems:
         - name: workload_metadata_config
           backend: local
+  - name: "deploy_service"
+    driver:
+      root_module_directory: test/fixtures/deploy_service
+    verifier:
+      systems:
+        - name: deploy_service
+          backend: local
+          controls:
+            - gcloud
+            - kubectl
+  - name: "node_pool"
+    driver:
+      root_module_directory: test/fixtures/node_pool
+    verifier:
+      systems:
+        - name: node_pool
+          backend: local
+          controls:
+            - gcloud
+            - kubectl
diff --git a/examples/disable_client_cert/README.md b/examples/disable_client_cert/README.md
index 2f531b9906..1b75e2b6e0 100644
--- a/examples/disable_client_cert/README.md
+++ b/examples/disable_client_cert/README.md
@@ -12,7 +12,6 @@ This example illustrates how to create a simple cluster and disable deprecated s
 |------|-------------|:----:|:-----:|:-----:|
 | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
-| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes |
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
 | network | The VPC network to host the cluster in | string | n/a | yes |
diff --git a/examples/disable_client_cert/variables.tf b/examples/disable_client_cert/variables.tf
index f1fdb25856..6c918f2344 100644
--- a/examples/disable_client_cert/variables.tf
+++ b/examples/disable_client_cert/variables.tf
@@ -18,10 +18,6 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "credentials_path" {
-  description = "The path to the GCP credentials JSON file"
-}
-
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""
diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf
index fc95090ede..7729c63166 100644
--- a/examples/simple_regional_beta/main.tf
+++ b/examples/simple_regional_beta/main.tf
@@ -19,9 +19,8 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
-  credentials = file(var.credentials_path)
-  region      = var.region
+  version = "~> 2.12.0"
+  region  = var.region
 }
 
 module "gke" {
diff --git a/examples/simple_regional_beta/test_outputs.tf b/examples/simple_regional_beta/test_outputs.tf
index f250fef192..e64c40e477 100644
--- a/examples/simple_regional_beta/test_outputs.tf
+++ b/examples/simple_regional_beta/test_outputs.tf
@@ -21,10 +21,6 @@ output "project_id" {
   value = var.project_id
 }
 
-output "credentials_path" {
-  value = var.credentials_path
-}
-
 output "region" {
   value = module.gke.region
 }
diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf
index 1da408a790..4ef289ed6b 100644
--- a/examples/simple_regional_beta/variables.tf
+++ b/examples/simple_regional_beta/variables.tf
@@ -18,10 +18,6 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "credentials_path" {
-  description = "The path to the GCP credentials JSON file"
-}
-
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""
diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf
index 0ca1873d86..e06567f960 100644
--- a/examples/simple_regional_private_beta/main.tf
+++ b/examples/simple_regional_private_beta/main.tf
@@ -19,9 +19,8 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.12.0"
-  credentials = file(var.credentials_path)
-  region      = var.region
+  version = "~> 2.12.0"
+  region  = var.region
 }
 
 data "google_compute_subnetwork" "subnetwork" {
diff --git a/examples/simple_regional_private_beta/test_outputs.tf b/examples/simple_regional_private_beta/test_outputs.tf
index 033c4beac1..53eab4ee12 100644
--- a/examples/simple_regional_private_beta/test_outputs.tf
+++ b/examples/simple_regional_private_beta/test_outputs.tf
@@ -21,10 +21,6 @@ output "project_id" {
   value = var.project_id
 }
 
-output "credentials_path" {
-  value = var.credentials_path
-}
-
 output "region" {
   value = module.gke.region
 }
diff --git a/examples/simple_regional_private_beta/variables.tf b/examples/simple_regional_private_beta/variables.tf
index 3fb7d8bab1..0ba1656629 100644
--- a/examples/simple_regional_private_beta/variables.tf
+++ b/examples/simple_regional_private_beta/variables.tf
@@ -18,10 +18,6 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
-variable "credentials_path" {
-  description = "The path to the GCP credentials JSON file"
-}
-
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""
diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf
index 428218d76a..e58097c560 100644
--- a/examples/simple_zonal_private/main.tf
+++ b/examples/simple_zonal_private/main.tf
@@ -15,7 +15,7 @@
  */
 
 locals {
-  cluster_type = "simple-regional-private"
+  cluster_type = "simple-zonal-private"
 }
 
 provider "google" {
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 42c9841fec..c9146e1826 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -15,6 +15,7 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
     export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi
diff --git a/test/ci/deploy-service.yml b/test/ci/deploy-service.yml
deleted file mode 100644
index 60505669b5..0000000000
--- a/test/ci/deploy-service.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "deploy-service-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/node-pool.yml b/test/ci/node-pool.yml
deleted file mode 100644
index 2402c32eba..0000000000
--- a/test/ci/node-pool.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "node-pool-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/shared-vpc.yml b/test/ci/shared-vpc.yml
deleted file mode 100644
index e209809246..0000000000
--- a/test/ci/shared-vpc.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "shared-vpc-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/simple-regional-private.yml b/test/ci/simple-regional-private.yml
deleted file mode 100644
index 36e00041d3..0000000000
--- a/test/ci/simple-regional-private.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "simple-regional-private-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/simple-regional.yml b/test/ci/simple-regional.yml
deleted file mode 100644
index a3bec48292..0000000000
--- a/test/ci/simple-regional.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "simple-regional-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/simple-zonal-private.yml b/test/ci/simple-zonal-private.yml
deleted file mode 100644
index 45dde32b64..0000000000
--- a/test/ci/simple-zonal-private.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "simple-zonal-private-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/simple-zonal.yml b/test/ci/simple-zonal.yml
deleted file mode 100644
index 6fcde7a1c7..0000000000
--- a/test/ci/simple-zonal.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "simple-zonal-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/stub-domains-upstream-nameservers.yml b/test/ci/stub-domains-upstream-nameservers.yml
deleted file mode 100644
index 4015338278..0000000000
--- a/test/ci/stub-domains-upstream-nameservers.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "stub-domains-upstream-nameservers-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/stub-domains.yml b/test/ci/stub-domains.yml
deleted file mode 100644
index c7130a3328..0000000000
--- a/test/ci/stub-domains.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "stub-domains-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/upstream-nameservers.yml b/test/ci/upstream-nameservers.yml
deleted file mode 100644
index 987884010a..0000000000
--- a/test/ci/upstream-nameservers.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "upstream-nameservers-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
diff --git a/test/ci/workload-metadata-config.yml b/test/ci/workload-metadata-config.yml
deleted file mode 100644
index 231c8dfc3a..0000000000
--- a/test/ci/workload-metadata-config.yml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-
-platform: linux
-
-inputs:
-- name: pull-request
-  path: terraform-google-kubernetes-engine
-
-run:
-  path: make
-  args: ['test_integration']
-  dir: terraform-google-kubernetes-engine
-
-params:
-  SUITE: "workload-metadata-config-local"
-  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
-  REGION: "us-east4"
-  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'
-
diff --git a/test/fixtures/disable_client_cert/example.tf b/test/fixtures/disable_client_cert/example.tf
index c1baed7c36..23ea6da936 100644
--- a/test/fixtures/disable_client_cert/example.tf
+++ b/test/fixtures/disable_client_cert/example.tf
@@ -18,7 +18,6 @@ module "example" {
   source = "../../../examples/disable_client_cert"
 
   project_id                     = var.project_id
-  credentials_path               = ""
   cluster_name_suffix            = "-${random_string.suffix.result}"
   region                         = var.region
   network                        = google_compute_network.main.name
diff --git a/test/integration/disable_client_cert/inspec.yml b/test/integration/disable_client_cert/inspec.yml
index 7959c6bb01..5803767d32 100644
--- a/test/integration/disable_client_cert/inspec.yml
+++ b/test/integration/disable_client_cert/inspec.yml
@@ -3,9 +3,6 @@ attributes:
   - name: project_id
     required: true
     type: string
-  - name: credentials_path
-    required: true
-    type: string
   - name: location
     required: true
     type: string
diff --git a/test/integration/simple_zonal/controls/gcp.rb b/test/integration/simple_zonal/controls/gcp.rb
index 6e9ade64ff..ad6ec1a7b2 100644
--- a/test/integration/simple_zonal/controls/gcp.rb
+++ b/test/integration/simple_zonal/controls/gcp.rb
@@ -24,7 +24,7 @@
     service_account_name = "projects/#{project_id}/serviceAccounts/#{service_account}"
   end
 
-  describe google_service_account name: service_account_name do
+  describe google_service_account(name: service_account_name) do
     its("display_name") { should eq "Terraform-managed service account for cluster #{attribute("cluster_name")}" }
     its("project_id") { should eq project_id }
   end
diff --git a/test/integration/simple_zonal/inspec.yml b/test/integration/simple_zonal/inspec.yml
index 5cb8ff9e01..3a3bc74145 100644
--- a/test/integration/simple_zonal/inspec.yml
+++ b/test/integration/simple_zonal/inspec.yml
@@ -2,7 +2,7 @@ name: simple_zonal
 depends:
   - name: inspec-gcp
     git: https://github.com/inspec/inspec-gcp.git
-    tag: v0.10.0
+    tag: v0.16.0
 attributes:
   - name: project_id
     required: true
@@ -25,7 +25,4 @@ attributes:
   - name: service_account
     required: true
     type: string
-  - name: service_account
-    required: true
-    type: string
 
diff --git a/test/integration/simple_zonal_private/inspec.yml b/test/integration/simple_zonal_private/inspec.yml
index ebe54e2374..4af03f2087 100644
--- a/test/integration/simple_zonal_private/inspec.yml
+++ b/test/integration/simple_zonal_private/inspec.yml
@@ -1,4 +1,4 @@
-name: simple_regional_private
+name: simple_zonal_private
 attributes:
   - name: project_id
     required: true
diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh
index 70ab3db5c8..ddfbab53c7 100755
--- a/test/task_helper_functions.sh
+++ b/test/task_helper_functions.sh
@@ -49,16 +49,3 @@ function check_generate() {
   rm -Rf "${tempdir}"
   return $((rval))
 }
-
-find_files() {
-  local pth="$1"
-  shift
-    find "${pth}" '(' \
-    -path '*/.git' -o \
-    -path '*/.terraform' -o \
-    -path '*/.kitchen' -o \
-    -path './autogen' -o \
-    -path './test/fixtures/all_examples' -o \
-    -path './test/fixtures/shared' ')' \
-    -prune -o -type f "$@"
-}

From 40a08a085ec0503994189c5e21c41f9d1589208c Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Mon, 21 Oct 2019 12:36:06 +0300
Subject: [PATCH 44/90] Updated dev tools docker image tag to 0.4.6

 * Fixed lint tests
 * Updated Makefile
 * Fixed check_generate test function
---
 Makefile                                         | 16 ++++++++--------
 autogen/scripts/wait-for-cluster.sh              |  3 ++-
 build/int.cloudbuild.yaml                        |  2 +-
 build/lint.cloudbuild.yaml                       |  2 +-
 .../scripts/wait-for-cluster.sh                  |  3 ++-
 .../scripts/wait-for-cluster.sh                  |  3 ++-
 .../scripts/wait-for-cluster.sh                  |  3 ++-
 .../scripts/wait-for-cluster.sh                  |  3 ++-
 .../private-cluster/scripts/wait-for-cluster.sh  |  3 ++-
 scripts/wait-for-cluster.sh                      |  5 +++--
 test/task_helper_functions.sh                    | 16 ++--------------
 11 files changed, 27 insertions(+), 32 deletions(-)

diff --git a/Makefile b/Makefile
index 52c9deaf8f..736cad34ce 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.5
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.6
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
@@ -27,7 +27,7 @@ REGISTRY_URL := gcr.io/cloud-foundation-cicd
 docker_run:
 	docker run --rm -it \
 		-e SERVICE_ACCOUNT_JSON \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/bin/bash
 
@@ -39,7 +39,7 @@ docker_test_prepare:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh prepare_environment
 
@@ -51,7 +51,7 @@ docker_test_cleanup:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh cleanup_environment
 
@@ -60,7 +60,7 @@ docker_test_cleanup:
 docker_test_integration:
 	docker run --rm -it \
 		-e SERVICE_ACCOUNT_JSON \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_integration.sh
 
@@ -68,7 +68,7 @@ docker_test_integration:
 .PHONY: docker_test_lint
 docker_test_lint:
 	docker run --rm -it \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_lint.sh
 
@@ -76,7 +76,7 @@ docker_test_lint:
 .PHONY: docker_generate_docs
 docker_generate_docs:
 	docker run --rm -it \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs'
 
@@ -84,7 +84,7 @@ docker_generate_docs:
 .PHONY: docker_generate
 docker_generate:
 	docker run --rm -it \
-                -v $(CURDIR):/workspace \
+                -v "$(CURDIR)":/workspace \
                 $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
                 /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate'
 
diff --git a/autogen/scripts/wait-for-cluster.sh b/autogen/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/autogen/scripts/wait-for-cluster.sh
+++ b/autogen/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 379205db45..4d77c4b1f1 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -226,4 +226,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml
index d5b2622e4f..7ba0827bdb 100644
--- a/build/lint.cloudbuild.yaml
+++ b/build/lint.cloudbuild.yaml
@@ -24,4 +24,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
diff --git a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/modules/beta-private-cluster/scripts/wait-for-cluster.sh b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/modules/beta-private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-private-cluster/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/modules/beta-public-cluster/scripts/wait-for-cluster.sh b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/modules/beta-public-cluster/scripts/wait-for-cluster.sh
+++ b/modules/beta-public-cluster/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/modules/private-cluster/scripts/wait-for-cluster.sh b/modules/private-cluster/scripts/wait-for-cluster.sh
index 7bf9919b8f..b7019eace1 100755
--- a/modules/private-cluster/scripts/wait-for-cluster.sh
+++ b/modules/private-cluster/scripts/wait-for-cluster.sh
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 42c9841fec..b7019eace1 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +15,9 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
-    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+    export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
 PROJECT=$1
diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh
index 70ab3db5c8..7de28bbdfe 100755
--- a/test/task_helper_functions.sh
+++ b/test/task_helper_functions.sh
@@ -32,8 +32,9 @@ function check_generate() {
     --exclude '*/.kitchen' \
     --exclude '*/.git' \
     /workspace "${tempdir}" >/dev/null 2>/dev/null
-  cd "${tempdir}" || exit 1
+  cd "${tempdir}/workspace" || exit 1
   generate >/dev/null 2>/dev/null
+  generate_docs >/dev/null 2>/dev/null
   diff -r \
     --exclude=".terraform" \
     --exclude=".kitchen" \
@@ -49,16 +50,3 @@ function check_generate() {
   rm -Rf "${tempdir}"
   return $((rval))
 }
-
-find_files() {
-  local pth="$1"
-  shift
-    find "${pth}" '(' \
-    -path '*/.git' -o \
-    -path '*/.terraform' -o \
-    -path '*/.kitchen' -o \
-    -path './autogen' -o \
-    -path './test/fixtures/all_examples' -o \
-    -path './test/fixtures/shared' ')' \
-    -prune -o -type f "$@"
-}

From 3933a794ddb7ca91f6fde1ea9d696b470c7e37c0 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Fri, 25 Oct 2019 11:44:44 -0400
Subject: [PATCH 45/90] Update CHANGELOG.md

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e609c0eca..efdebd5e9e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 ## [Unreleased]
 
+### Fixed
+
+* Fixed bug with setting up sandboxing on nodes. [#286]
+
 ## [v5.1.0] - 2019-10-24
 
 ### Added
@@ -214,6 +218,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
+[#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286
 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285
 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284
 [#282]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/282

From ab02f24887a8e8adffbea589f732b5a15d34af93 Mon Sep 17 00:00:00 2001
From: Aaron Lane <aarondrewlane@gmail.com>
Date: Fri, 25 Oct 2019 14:07:19 -0400
Subject: [PATCH 46/90] Add 5.1.1 to CHANGELOG

---
 CHANGELOG.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index efdebd5e9e..cf6ef22df2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 ## [Unreleased]
 
+## [v5.1.1] - 2019-10-25
+
 ### Fixed
 
 * Fixed bug with setting up sandboxing on nodes. [#286]
@@ -202,7 +204,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 
 * Initial release of module.
 
-[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...HEAD
+[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...HEAD
+[v5.1.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...v5.1.1
 [v5.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...v5.1.0
 [v5.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.1.0...v5.0.0
 [v4.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.0.0...v4.1.0

From 34197c263e23f18096e931aad133d5aaad734020 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Sat, 26 Oct 2019 13:16:17 -0500
Subject: [PATCH 47/90] add desc, new ci steps, unused resources

---
 build/int.cloudbuild.yaml                     | 20 +++++++++++++++++++
 .../simple_regional_with_networking/README.md |  6 +++---
 .../simple_regional_with_networking/main.tf   |  5 -----
 .../outputs.tf                                | 13 +++++++-----
 .../example.tf                                |  6 ------
 5 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index a4ce07ab4a..8bf89efe09 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -101,6 +101,26 @@ steps:
     - verify simple-regional-private-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-private-local']
+- id: create simple-regional-with-networking-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-with-networking-local']
+- id: converge simple-regional-with-networking-local
+  waitFor:
+    - create simple-regional-with-networking-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-with-networking-local']
+- id: verify simple-regional-with-networking-local
+  waitFor:
+    - converge simple-regional-with-networking-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-with-networking-local']
+- id: destroy simple-regional-with-networking-local
+  waitFor:
+    - verify simple-regional-with-networking-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-networking-local']
 - id: create simple-zonal-local
   waitFor:
     - prepare
diff --git a/examples/simple_regional_with_networking/README.md b/examples/simple_regional_with_networking/README.md
index 2f9d844a61..8ef0dad0ee 100644
--- a/examples/simple_regional_with_networking/README.md
+++ b/examples/simple_regional_with_networking/README.md
@@ -19,12 +19,12 @@ This example illustrates how to create a VPC and a simple cluster.
 
 | Name | Description |
 |------|-------------|
-| ca\_certificate |  |
-| client\_token |  |
+| ca\_certificate | The cluster ca certificate (base64 encoded) |
+| client\_token | The bearer token for auth |
 | cluster\_name | Cluster name |
 | ip\_range\_pods\_name | The secondary IP range used for pods |
 | ip\_range\_services\_name | The secondary IP range used for services |
-| kubernetes\_endpoint |  |
+| kubernetes\_endpoint | The cluster endpoint |
 | location |  |
 | master\_kubernetes\_version | The master Kubernetes version |
 | network |  |
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index 7789d73cdd..7b39615114 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -14,11 +14,6 @@
  * limitations under the License.
  */
 
-provider "google" {
-  version = "~> 2.12.0"
-  region  = var.region
-}
-
 module "gcp-network" {
   source       = "terraform-google-modules/network/google"
   version      = "~> 1.4.0"
diff --git a/examples/simple_regional_with_networking/outputs.tf b/examples/simple_regional_with_networking/outputs.tf
index a26f0275e0..bb255b54a2 100644
--- a/examples/simple_regional_with_networking/outputs.tf
+++ b/examples/simple_regional_with_networking/outputs.tf
@@ -15,17 +15,20 @@
  */
 
 output "kubernetes_endpoint" {
-  sensitive = true
-  value     = module.gke.endpoint
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
 }
 
 output "client_token" {
-  sensitive = true
-  value     = base64encode(data.google_client_config.default.access_token)
+  description = "The bearer token for auth"
+  sensitive   = true
+  value       = base64encode(data.google_client_config.default.access_token)
 }
 
 output "ca_certificate" {
-  value = module.gke.ca_certificate
+  description = "The cluster ca certificate (base64 encoded)"
+  value       = module.gke.ca_certificate
 }
 
 output "service_account" {
diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf
index ace90efb3d..c7ae5af76c 100644
--- a/test/fixtures/simple_regional_with_networking/example.tf
+++ b/test/fixtures/simple_regional_with_networking/example.tf
@@ -14,12 +14,6 @@
  * limitations under the License.
  */
 
-resource "random_string" "suffix" {
-  length  = 4
-  special = false
-  upper   = false
-}
-
 module "example" {
   source = "../../../examples/simple_regional_with_networking"
 

From 0fd2555cfee00e8daf87d1903d09b5b5f0b62f53 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Tue, 29 Oct 2019 07:15:42 +0800
Subject: [PATCH 48/90] fix example

---
 examples/simple_regional_beta/main.tf | 5 ++---
 scripts/wait-for-cluster.sh           | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf
index c2e29b1298..0863cc51de 100644
--- a/examples/simple_regional_beta/main.tf
+++ b/examples/simple_regional_beta/main.tf
@@ -19,9 +19,8 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.18.0"
-  credentials = file(var.credentials_path)
-  region      = var.region
+  version = "~> 2.18.0"
+  region  = var.region
 }
 
 module "gke" {
diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh
index 42c9841fec..7bf9919b8f 100755
--- a/scripts/wait-for-cluster.sh
+++ b/scripts/wait-for-cluster.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From e45d9125c505b27441ad9f974fd80f73ec3827c3 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Tue, 29 Oct 2019 12:14:03 +0800
Subject: [PATCH 49/90] Remove enable_release_channel

---
 autogen/cluster.tf                                       | 8 ++------
 autogen/main.tf                                          | 4 ----
 autogen/outputs.tf                                       | 5 -----
 autogen/variables.tf                                     | 6 ------
 main.tf                                                  | 1 -
 modules/beta-private-cluster-update-variant/README.md    | 2 --
 modules/beta-private-cluster-update-variant/cluster.tf   | 8 ++------
 modules/beta-private-cluster-update-variant/main.tf      | 2 --
 modules/beta-private-cluster-update-variant/outputs.tf   | 5 -----
 modules/beta-private-cluster-update-variant/variables.tf | 6 ------
 modules/beta-private-cluster/README.md                   | 2 --
 modules/beta-private-cluster/cluster.tf                  | 8 ++------
 modules/beta-private-cluster/main.tf                     | 2 --
 modules/beta-private-cluster/outputs.tf                  | 5 -----
 modules/beta-private-cluster/variables.tf                | 6 ------
 modules/beta-public-cluster/README.md                    | 2 --
 modules/beta-public-cluster/cluster.tf                   | 8 ++------
 modules/beta-public-cluster/main.tf                      | 2 --
 modules/beta-public-cluster/outputs.tf                   | 5 -----
 modules/beta-public-cluster/variables.tf                 | 6 ------
 modules/private-cluster-update-variant/main.tf           | 1 -
 modules/private-cluster/main.tf                          | 1 -
 22 files changed, 8 insertions(+), 87 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 23d9a6b5ff..0742fd1523 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -46,12 +46,8 @@ resource "google_container_cluster" "primary" {
   }
 
 {% if beta_cluster %}
-  dynamic "release_channel" {
-    for_each = local.release_channel
-
-    content {
-      channel = release_channel.value.channel
-    }
+  release_channel {
+    channel = var.release_channel
   }
 {% endif %}
 
diff --git a/autogen/main.tf b/autogen/main.tf
index 9ed92d9741..afbd7bf8c1 100644
--- a/autogen/main.tf
+++ b/autogen/main.tf
@@ -48,10 +48,6 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
-{% if beta_cluster %}
-  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
-{% endif %}
-
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/autogen/outputs.tf b/autogen/outputs.tf
index 2f3e73fbb0..704569d00e 100644
--- a/autogen/outputs.tf
+++ b/autogen/outputs.tf
@@ -150,11 +150,6 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
-output "release_channel_enabled" {
-  description = "Whether release channel is enabled"
-  value       = var.enable_release_channel
-}
-
 output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
diff --git a/autogen/variables.tf b/autogen/variables.tf
index f24605d906..cbc9a9b4d6 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -422,12 +422,6 @@ variable "authenticator_security_group" {
   default     = null
 }
 
-variable "enable_release_channel" {
-  type        = bool
-  description = "(Beta) Whether release channel is configured for this cluster."
-  default     = false
-}
-
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
diff --git a/main.tf b/main.tf
index 1090227fd8..a9e1c15810 100644
--- a/main.tf
+++ b/main.tf
@@ -45,7 +45,6 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index bbe27e3964..fda632e3c4 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -153,7 +153,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -228,7 +227,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | release\_channel | The release channel of this cluster |
-| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 366280d7b8..8624bdd4ec 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -41,12 +41,8 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  dynamic "release_channel" {
-    for_each = local.release_channel
-
-    content {
-      channel = release_channel.value.channel
-    }
+  release_channel {
+    channel = var.release_channel
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index 760209d6df..63bf31ac78 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -44,8 +44,6 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
-  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
-
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf
index 31a22b2175..956c8c2d5d 100644
--- a/modules/beta-private-cluster-update-variant/outputs.tf
+++ b/modules/beta-private-cluster-update-variant/outputs.tf
@@ -149,11 +149,6 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
-output "release_channel_enabled" {
-  description = "Whether release channel is enabled"
-  value       = var.enable_release_channel
-}
-
 output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 8df35ee512..4d403b2b30 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -417,12 +417,6 @@ variable "authenticator_security_group" {
   default     = null
 }
 
-variable "enable_release_channel" {
-  type        = bool
-  description = "(Beta) Whether release channel is configured for this cluster."
-  default     = false
-}
-
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 4d04e587e7..f56c6957ed 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -153,7 +153,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -228,7 +227,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | release\_channel | The release channel of this cluster |
-| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index d0cc6d7c20..2c29d8b81f 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -41,12 +41,8 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  dynamic "release_channel" {
-    for_each = local.release_channel
-
-    content {
-      channel = release_channel.value.channel
-    }
+  release_channel {
+    channel = var.release_channel
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
index 760209d6df..63bf31ac78 100644
--- a/modules/beta-private-cluster/main.tf
+++ b/modules/beta-private-cluster/main.tf
@@ -44,8 +44,6 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
-  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
-
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf
index 31a22b2175..956c8c2d5d 100644
--- a/modules/beta-private-cluster/outputs.tf
+++ b/modules/beta-private-cluster/outputs.tf
@@ -149,11 +149,6 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
-output "release_channel_enabled" {
-  description = "Whether release channel is enabled"
-  value       = var.enable_release_channel
-}
-
 output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index 8df35ee512..4d403b2b30 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -417,12 +417,6 @@ variable "authenticator_security_group" {
   default     = null
 }
 
-variable "enable_release_channel" {
-  type        = bool
-  description = "(Beta) Whether release channel is configured for this cluster."
-  default     = false
-}
-
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 81b887dac9..18697cf134 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -145,7 +145,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_release\_channel | (Beta) Whether release channel is configured for this cluster. | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
@@ -219,7 +218,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | pod\_security\_policy\_enabled | Whether pod security policy is enabled |
 | region | Cluster region |
 | release\_channel | The release channel of this cluster |
-| release\_channel\_enabled | Whether release channel is enabled |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
 | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 304fcc8df3..073affb113 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -41,12 +41,8 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  dynamic "release_channel" {
-    for_each = local.release_channel
-
-    content {
-      channel = release_channel.value.channel
-    }
+  release_channel {
+    channel = var.release_channel
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf
index ec0afeda3b..f0477f9498 100644
--- a/modules/beta-public-cluster/main.tf
+++ b/modules/beta-public-cluster/main.tf
@@ -44,8 +44,6 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
-  release_channel         = var.enable_release_channel ? [{ channel : var.release_channel }] : []
-
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf
index 31a22b2175..956c8c2d5d 100644
--- a/modules/beta-public-cluster/outputs.tf
+++ b/modules/beta-public-cluster/outputs.tf
@@ -149,11 +149,6 @@ output "vertical_pod_autoscaling_enabled" {
   value       = local.cluster_vertical_pod_autoscaling_enabled
 }
 
-output "release_channel_enabled" {
-  description = "Whether release channel is enabled"
-  value       = var.enable_release_channel
-}
-
 output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 9bf971416a..ae89886ea7 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -393,12 +393,6 @@ variable "authenticator_security_group" {
   default     = null
 }
 
-variable "enable_release_channel" {
-  type        = bool
-  description = "(Beta) Whether release channel is configured for this cluster."
-  default     = false
-}
-
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index aba5e2d79f..2bd1c40d14 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -45,7 +45,6 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
index aba5e2d79f..2bd1c40d14 100644
--- a/modules/private-cluster/main.tf
+++ b/modules/private-cluster/main.tf
@@ -45,7 +45,6 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

From 42a0376309903e225c7811ff7bc3a35e8685d908 Mon Sep 17 00:00:00 2001
From: Chris Sng <chrissng@users.noreply.github.com>
Date: Tue, 29 Oct 2019 14:05:05 +0800
Subject: [PATCH 50/90] exclude release_channel block if null

---
 autogen/cluster.tf                                       | 8 ++++++--
 autogen/main.tf                                          | 4 ++++
 autogen/variables.tf                                     | 2 +-
 main.tf                                                  | 1 +
 modules/beta-private-cluster-update-variant/README.md    | 2 +-
 modules/beta-private-cluster-update-variant/cluster.tf   | 8 ++++++--
 modules/beta-private-cluster-update-variant/main.tf      | 2 ++
 modules/beta-private-cluster-update-variant/variables.tf | 2 +-
 modules/beta-private-cluster/README.md                   | 2 +-
 modules/beta-private-cluster/cluster.tf                  | 8 ++++++--
 modules/beta-private-cluster/main.tf                     | 2 ++
 modules/beta-private-cluster/variables.tf                | 2 +-
 modules/beta-public-cluster/README.md                    | 2 +-
 modules/beta-public-cluster/cluster.tf                   | 8 ++++++--
 modules/beta-public-cluster/main.tf                      | 2 ++
 modules/beta-public-cluster/variables.tf                 | 2 +-
 modules/private-cluster-update-variant/main.tf           | 1 +
 modules/private-cluster/main.tf                          | 1 +
 18 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 0742fd1523..23d9a6b5ff 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -46,8 +46,12 @@ resource "google_container_cluster" "primary" {
   }
 
 {% if beta_cluster %}
-  release_channel {
-    channel = var.release_channel
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
   }
 {% endif %}
 
diff --git a/autogen/main.tf b/autogen/main.tf
index afbd7bf8c1..30347b9b15 100644
--- a/autogen/main.tf
+++ b/autogen/main.tf
@@ -48,6 +48,10 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+{% if beta_cluster %}
+  release_channel         = var.release_channel != null ? [{ channel : var.release_channel }] : []
+{% endif %}
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/autogen/variables.tf b/autogen/variables.tf
index cbc9a9b4d6..3f2a12f3a9 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -425,6 +425,6 @@ variable "authenticator_security_group" {
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
-  default     = "UNSPECIFIED"
+  default     = null
 }
 {% endif %}
diff --git a/main.tf b/main.tf
index a9e1c15810..1090227fd8 100644
--- a/main.tf
+++ b/main.tf
@@ -45,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index fda632e3c4..7a3be69fc4 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -191,7 +191,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 8624bdd4ec..366280d7b8 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -41,8 +41,12 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  release_channel {
-    channel = var.release_channel
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index 63bf31ac78..2de95c063d 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -44,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.release_channel != null ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 4d403b2b30..07461351c1 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -420,5 +420,5 @@ variable "authenticator_security_group" {
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
-  default     = "UNSPECIFIED"
+  default     = null
 }
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index f56c6957ed..daf98949a1 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -191,7 +191,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 2c29d8b81f..d0cc6d7c20 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -41,8 +41,12 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  release_channel {
-    channel = var.release_channel
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
index 63bf31ac78..2de95c063d 100644
--- a/modules/beta-private-cluster/main.tf
+++ b/modules/beta-private-cluster/main.tf
@@ -44,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.release_channel != null ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index 4d403b2b30..07461351c1 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -420,5 +420,5 @@ variable "authenticator_security_group" {
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
-  default     = "UNSPECIFIED"
+  default     = null
 }
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 18697cf134..98f4526d9f 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -182,7 +182,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"UNSPECIFIED"` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 073affb113..304fcc8df3 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -41,8 +41,12 @@ resource "google_container_cluster" "primary" {
     }
   }
 
-  release_channel {
-    channel = var.release_channel
+  dynamic "release_channel" {
+    for_each = local.release_channel
+
+    content {
+      channel = release_channel.value.channel
+    }
   }
 
   subnetwork         = data.google_compute_subnetwork.gke_subnetwork.self_link
diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf
index f0477f9498..9668b6f1ea 100644
--- a/modules/beta-public-cluster/main.tf
+++ b/modules/beta-public-cluster/main.tf
@@ -44,6 +44,8 @@ locals {
   node_version_zonal      = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
+  release_channel         = var.release_channel != null ? [{ channel : var.release_channel }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index ae89886ea7..b41e5591b9 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -396,5 +396,5 @@ variable "authenticator_security_group" {
 variable "release_channel" {
   type        = string
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
-  default     = "UNSPECIFIED"
+  default     = null
 }
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index 2bd1c40d14..aba5e2d79f 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -45,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id
diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
index 2bd1c40d14..aba5e2d79f 100644
--- a/modules/private-cluster/main.tf
+++ b/modules/private-cluster/main.tf
@@ -45,6 +45,7 @@ locals {
   master_version          = var.regional ? local.master_version_regional : local.master_version_zonal
   node_version            = var.regional ? local.node_version_regional : local.node_version_zonal
 
+
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

From eadb54a569695df65b53ea5411484341e55c24ee Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Tue, 29 Oct 2019 14:34:35 +0200
Subject: [PATCH 51/90] Added steps for `deploy-service-local` and
 `node-pool-local`

---
 build/int.cloudbuild.yaml | 40 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 379205db45..d397b703e3 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -221,6 +221,46 @@ steps:
     - verify workload-metadata-config-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local']
+- id: create deploy-service-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create deploy-service-local']
+- id: converge deploy-service-local
+  waitFor:
+    - create deploy-service-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge deploy-service-local']
+- id: verify deploy-service-local
+  waitFor:
+    - converge deploy-service-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify deploy-service-local']
+- id: destroy deploy-service-local
+  waitFor:
+    - verify deploy-service-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy deploy-service-local']
+- id: create node-pool-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create node-pool-local']
+- id: converge node-pool-local
+  waitFor:
+    - create node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local']
+- id: verify node-pool-local
+  waitFor:
+    - converge node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local']
+- id: destroy node-pool-local
+  waitFor:
+    - verify node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local']
 tags:
 - 'ci'
 - 'integration'

From 9720e805f6562bd931084e43439456ed4b8f36b2 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Tue, 29 Oct 2019 14:40:53 +0200
Subject: [PATCH 52/90] Minor grammatical corrections

---
 autogen/README.md             | 96 -----------------------------------
 modules/acm/README.md         |  6 +--
 test/task_helper_functions.sh |  2 +-
 3 files changed, 3 insertions(+), 101 deletions(-)

diff --git a/autogen/README.md b/autogen/README.md
index 14d7409c7f..7b854e0351 100644
--- a/autogen/README.md
+++ b/autogen/README.md
@@ -139,102 +139,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
-| region | Cluster region |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/acm/README.md b/modules/acm/README.md
index 40fdb939db..8bc8abc9f0 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -15,10 +15,8 @@ See [examples/simple_zonal](../../examples/simple_zonal) cluster example.
 
 ## Installation
 
-Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page.
-Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public**
-output variable to your account at your Git server.
-
+This module automates the instructions described in the [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) guide.
+To enable Git access to the configuration repository over SSH, complete step 2 in the [Using an SSH keypair](https://cloud.google.com/anthos-config-management/docs/how-to/installing#git-creds-ssh) section using the SSH public key from the **git\_creds\_public** output.
 
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh
index ddedde4b4c..70ab3db5c8 100755
--- a/test/task_helper_functions.sh
+++ b/test/task_helper_functions.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash{
+#!/usr/bin/env bash
 
 # Copyright 2019 Google LLC
 #

From fd2230a516b634e68dac0c0f543ea4bfe4d26e8e Mon Sep 17 00:00:00 2001
From: Bohdan Yurov <bohdan.yurov@globallogic.com>
Date: Thu, 24 Oct 2019 19:15:32 +0200
Subject: [PATCH 53/90] Fixes #261: Invalid plan when re-applying terraform
 module
 https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/261

Bumbed minimal provider version to 2.18
---
 autogen/versions.tf                                     | 8 ++++++++
 examples/deploy_service/main.tf                         | 2 +-
 examples/disable_client_cert/main.tf                    | 2 +-
 examples/node_pool_update_variant/main.tf               | 2 +-
 examples/shared_vpc/main.tf                             | 2 +-
 examples/simple_regional/main.tf                        | 2 +-
 examples/simple_regional_private/main.tf                | 2 +-
 examples/simple_zonal/main.tf                           | 2 +-
 examples/simple_zonal_private/main.tf                   | 2 +-
 examples/stub_domains/main.tf                           | 2 +-
 examples/stub_domains_private/main.tf                   | 6 +-----
 examples/stub_domains_upstream_nameservers/main.tf      | 2 +-
 examples/upstream_nameservers/main.tf                   | 2 +-
 modules/beta-private-cluster-update-variant/versions.tf | 4 ++++
 modules/beta-private-cluster/versions.tf                | 4 ++++
 modules/beta-public-cluster/versions.tf                 | 4 ++++
 modules/private-cluster-update-variant/versions.tf      | 4 ++++
 modules/private-cluster/versions.tf                     | 4 ++++
 versions.tf                                             | 4 ++++
 19 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/autogen/versions.tf b/autogen/versions.tf
index 832ec1df39..387a2e37c8 100644
--- a/autogen/versions.tf
+++ b/autogen/versions.tf
@@ -16,4 +16,12 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+{% if beta_cluster %}
+    google-beta = "~> 2.18.0"
+{% else %}
+    google = "~> 2.18.0"
+{% endif %}
+  }
 }
diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf
index 3e15164913..a61aa9c9b8 100644
--- a/examples/deploy_service/main.tf
+++ b/examples/deploy_service/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/disable_client_cert/main.tf b/examples/disable_client_cert/main.tf
index 221eb994ce..4d040ffcb7 100644
--- a/examples/disable_client_cert/main.tf
+++ b/examples/disable_client_cert/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf
index c10e797511..9b29a5f0fe 100644
--- a/examples/node_pool_update_variant/main.tf
+++ b/examples/node_pool_update_variant/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf
index 766239ead8..3533130a4e 100644
--- a/examples/shared_vpc/main.tf
+++ b/examples/shared_vpc/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf
index 353ae91906..772cf8d0e5 100644
--- a/examples/simple_regional/main.tf
+++ b/examples/simple_regional/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf
index 89568e86ee..f17a3728a6 100644
--- a/examples/simple_regional_private/main.tf
+++ b/examples/simple_regional_private/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf
index edd90f7a0d..3490252a4a 100644
--- a/examples/simple_zonal/main.tf
+++ b/examples/simple_zonal/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf
index 428218d76a..7a3db12157 100644
--- a/examples/simple_zonal_private/main.tf
+++ b/examples/simple_zonal_private/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf
index 37264f781b..b81dc0cf8c 100644
--- a/examples/stub_domains/main.tf
+++ b/examples/stub_domains/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf
index 65bc48247d..3f268e75a4 100644
--- a/examples/stub_domains_private/main.tf
+++ b/examples/stub_domains_private/main.tf
@@ -15,14 +15,10 @@
  */
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
-provider "random" {
-  version = "~> 2.1"
-}
-
 data "google_compute_subnetwork" "subnetwork" {
   name    = var.subnetwork
   project = var.project_id
diff --git a/examples/stub_domains_upstream_nameservers/main.tf b/examples/stub_domains_upstream_nameservers/main.tf
index 0da83b95c6..4b7448b7e8 100644
--- a/examples/stub_domains_upstream_nameservers/main.tf
+++ b/examples/stub_domains_upstream_nameservers/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/examples/upstream_nameservers/main.tf b/examples/upstream_nameservers/main.tf
index ecded7c29c..784e8a0cd3 100644
--- a/examples/upstream_nameservers/main.tf
+++ b/examples/upstream_nameservers/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 2.12.0"
+  version = "~> 2.18.0"
   region  = var.region
 }
 
diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf
index 832ec1df39..8e29303fa9 100644
--- a/modules/beta-private-cluster-update-variant/versions.tf
+++ b/modules/beta-private-cluster-update-variant/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google-beta = "~> 2.18.0"
+  }
 }
diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf
index 832ec1df39..8e29303fa9 100644
--- a/modules/beta-private-cluster/versions.tf
+++ b/modules/beta-private-cluster/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google-beta = "~> 2.18.0"
+  }
 }
diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf
index 832ec1df39..8e29303fa9 100644
--- a/modules/beta-public-cluster/versions.tf
+++ b/modules/beta-public-cluster/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google-beta = "~> 2.18.0"
+  }
 }
diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf
index 832ec1df39..e4544656fa 100644
--- a/modules/private-cluster-update-variant/versions.tf
+++ b/modules/private-cluster-update-variant/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google = "~> 2.18.0"
+  }
 }
diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf
index 832ec1df39..e4544656fa 100644
--- a/modules/private-cluster/versions.tf
+++ b/modules/private-cluster/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google = "~> 2.18.0"
+  }
 }
diff --git a/versions.tf b/versions.tf
index 832ec1df39..e4544656fa 100644
--- a/versions.tf
+++ b/versions.tf
@@ -16,4 +16,8 @@
 
 terraform {
   required_version = ">= 0.12"
+
+  required_providers {
+    google = "~> 2.18.0"
+  }
 }

From 983952558b54c8d3f6c0c271517d330efaf04302 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Wed, 30 Oct 2019 18:58:52 +0200
Subject: [PATCH 54/90] Fix `gsutil` credentials

---
 modules/acm/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 10e55e9f88..117e13c94b 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" {
 
 resource "null_resource" "acm_operator_config" {
   provisioner "local-exec" {
-    command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
+    command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
   }
 
   provisioner "local-exec" {

From 8ab27142736cd5ef59ad2660cf8fae2d6483b9f3 Mon Sep 17 00:00:00 2001
From: Devan Patel <devan2patel@gmail.com>
Date: Thu, 31 Oct 2019 11:07:13 +0000
Subject: [PATCH 55/90] Add enable_shielded_nodes beta field

---
 autogen/cluster.tf                                       | 1 +
 autogen/variables.tf                                     | 6 ++++++
 modules/beta-private-cluster-update-variant/README.md    | 1 +
 modules/beta-private-cluster-update-variant/cluster.tf   | 1 +
 modules/beta-private-cluster-update-variant/variables.tf | 6 ++++++
 modules/beta-private-cluster/README.md                   | 1 +
 modules/beta-private-cluster/cluster.tf                  | 1 +
 modules/beta-private-cluster/variables.tf                | 6 ++++++
 modules/beta-public-cluster/README.md                    | 1 +
 modules/beta-public-cluster/cluster.tf                   | 1 +
 modules/beta-public-cluster/variables.tf                 | 6 ++++++
 11 files changed, 31 insertions(+)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 23d9a6b5ff..0bac34a37e 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -65,6 +65,7 @@ resource "google_container_cluster" "primary" {
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
   default_max_pods_per_node   = var.default_max_pods_per_node
+  enable_shielded_nodes       = var.enable_shielded_nodes
 
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
diff --git a/autogen/variables.tf b/autogen/variables.tf
index 3f2a12f3a9..ad5cc44e34 100644
--- a/autogen/variables.tf
+++ b/autogen/variables.tf
@@ -427,4 +427,10 @@ variable "release_channel" {
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
   default     = null
 }
+
+variable "enable_shielded_nodes" {
+  type = bool
+  description = "Enable Shielded Nodes features on all nodes in this cluster"
+  default = false
+}
 {% endif %}
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 7a3be69fc4..93daea71ca 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -153,6 +153,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 366280d7b8..6039cecd98 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -58,6 +58,7 @@ resource "google_container_cluster" "primary" {
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
   default_max_pods_per_node   = var.default_max_pods_per_node
+  enable_shielded_nodes       = var.enable_shielded_nodes
 
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index 07461351c1..d04ed5ac1f 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -422,3 +422,9 @@ variable "release_channel" {
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
   default     = null
 }
+
+variable "enable_shielded_nodes" {
+  type        = bool
+  description = "Enable Shielded Nodes features on all nodes in this cluster"
+  default     = false
+}
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index daf98949a1..4096848139 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -153,6 +153,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index d0cc6d7c20..10e12a9ba0 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -58,6 +58,7 @@ resource "google_container_cluster" "primary" {
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
   default_max_pods_per_node   = var.default_max_pods_per_node
+  enable_shielded_nodes       = var.enable_shielded_nodes
 
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index 07461351c1..d04ed5ac1f 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -422,3 +422,9 @@ variable "release_channel" {
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
   default     = null
 }
+
+variable "enable_shielded_nodes" {
+  type        = bool
+  description = "Enable Shielded Nodes features on all nodes in this cluster"
+  default     = false
+}
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 98f4526d9f..258ea431e4 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -145,6 +145,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 304fcc8df3..b5f896bc1b 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -58,6 +58,7 @@ resource "google_container_cluster" "primary" {
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
   default_max_pods_per_node   = var.default_max_pods_per_node
+  enable_shielded_nodes       = var.enable_shielded_nodes
 
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index b41e5591b9..a1057f1843 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -398,3 +398,9 @@ variable "release_channel" {
   description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`."
   default     = null
 }
+
+variable "enable_shielded_nodes" {
+  type        = bool
+  description = "Enable Shielded Nodes features on all nodes in this cluster"
+  default     = false
+}

From 28f2470742fdfc477b437cf9ced4afc52dbb256e Mon Sep 17 00:00:00 2001
From: Devan Patel <devan2patel@gmail.com>
Date: Thu, 31 Oct 2019 12:02:31 +0000
Subject: [PATCH 56/90] Changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf6ef22df2..74a293e683 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 ## [Unreleased]
 
+### Added
+
+* Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300]
+
 ## [v5.1.1] - 2019-10-25
 
 ### Fixed

From 6b13a01f16134872b4cadf42fa0df3f2c9e6457b Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 31 Oct 2019 18:38:33 -0500
Subject: [PATCH 57/90] add node pool location, enable and update test

---
 .kitchen.yml                                  | 16 ++--
 autogen/cluster.tf                            |  4 +
 examples/node_pool/main.tf                    | 17 +++-
 .../cluster.tf                                |  4 +-
 modules/beta-private-cluster/cluster.tf       |  4 +-
 modules/beta-public-cluster/cluster.tf        |  4 +-
 test/integration/node_pool/controls/gcloud.rb | 92 ++++++++++++++++++-
 7 files changed, 126 insertions(+), 15 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index 81603782cd..b5471f7e49 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -45,15 +45,13 @@ suites:
       systems:
         - name: disable_client_cert
           backend: local
-# Disabled due to issue #274
-# (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/274)
-#  - name: "node_pool"
-#    driver:
-#      root_module_directory: test/fixtures/node_pool
-#    verifier:
-#      systems:
-#        - name: node_pool
-#          backend: local
+  - name: "node_pool"
+    driver:
+      root_module_directory: test/fixtures/node_pool
+    verifier:
+      systems:
+        - name: node_pool
+          backend: local
   - name: "shared_vpc"
     driver:
       root_module_directory: test/fixtures/shared_vpc
diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 0bac34a37e..2bd52acea6 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -310,6 +310,10 @@ resource "google_container_node_pool" "pools" {
   {% endif %}
   project  = var.project_id
   location = local.location
+  {% if beta_cluster %}
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  {% endif %}
   cluster  = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index c7a7f852ae..3120c5735f 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -27,7 +27,6 @@ module "gke" {
   source                            = "../../modules/beta-public-cluster/"
   project_id                        = var.project_id
   name                              = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  regional                          = false
   region                            = var.region
   zones                             = var.zones
   network                           = var.network
@@ -59,12 +58,24 @@ module "gke" {
       auto_repair       = false
       service_account   = var.compute_engine_service_account
     },
+    {
+      name            = "pool-03"
+      node_locations  = "us-east4-b,us-east4-c"
+      machine_type    = "n1-standard-2"
+      min_count       = 1
+      max_count       = 2
+      disk_type       = "pd-standard"
+      image_type      = "COS"
+      auto_upgrade    = true
+      service_account = var.compute_engine_service_account
+    },
   ]
 
   node_pools_oauth_scopes = {
     all     = []
     pool-01 = []
     pool-02 = []
+    pool-03 = []
   }
 
   node_pools_metadata = {
@@ -73,6 +84,7 @@ module "gke" {
       shutdown-script = file("${path.module}/data/shutdown-script.sh")
     }
     pool-02 = {}
+    pool-03 = {}
   }
 
   node_pools_labels = {
@@ -83,6 +95,7 @@ module "gke" {
       pool-01-example = true
     }
     pool-02 = {}
+    pool-03 = {}
   }
 
   node_pools_taints = {
@@ -101,6 +114,7 @@ module "gke" {
       },
     ]
     pool-02 = []
+    pool-03 = []
   }
 
   node_pools_tags = {
@@ -111,6 +125,7 @@ module "gke" {
       "pool-01-example",
     ]
     pool-02 = []
+    pool-03 = []
   }
 }
 
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 6039cecd98..bc9d349205 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -284,7 +284,9 @@ resource "google_container_node_pool" "pools" {
   name     = random_id.name.*.hex[count.index]
   project  = var.project_id
   location = local.location
-  cluster  = google_container_cluster.primary.name
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
     "version",
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 10e12a9ba0..2aa7521203 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -212,7 +212,9 @@ resource "google_container_node_pool" "pools" {
   name     = var.node_pools[count.index]["name"]
   project  = var.project_id
   location = local.location
-  cluster  = google_container_cluster.primary.name
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
     "version",
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index b5f896bc1b..24ec6f9b35 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -207,7 +207,9 @@ resource "google_container_node_pool" "pools" {
   name     = var.node_pools[count.index]["name"]
   project  = var.project_id
   location = local.location
-  cluster  = google_container_cluster.primary.name
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
     "version",
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
index 6ff5fdd201..170827f2d5 100644
--- a/test/integration/node_pool/controls/gcloud.rb
+++ b/test/integration/node_pool/controls/gcloud.rb
@@ -36,8 +36,8 @@
     describe "node pools" do
       let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } }
 
-      it "has 2" do
-        expect(node_pools.count).to eq 2
+      it "has 3" do
+        expect(node_pools.count).to eq 3
       end
 
       describe "pool-01" do
@@ -279,6 +279,94 @@
           )
         end
       end
+      describe "pool-03" do
+        it "exists" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+            )
+          )
+        end
+        it "is the expected machine type" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "machineType" => "n1-standard-2",
+              ),
+            )
+          )
+        end
+
+        it "has autoscaling enabled" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "autoscaling" => including(
+                "enabled" => true,
+              ),
+            )
+          )
+        end
+
+        it "has the expected minimum node count" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "autoscaling" => including(
+                "minNodeCount" => 1,
+              ),
+            )
+          )
+        end
+
+        it "has autorepair enabled" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "management" => including(
+                "autoRepair" => true,
+              ),
+            )
+          )
+        end
+
+        it "has automatic upgrades enabled" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "management" => including(
+                "autoUpgrade" => true,
+              ),
+            )
+          )
+        end
+
+      end
+    end
+  end
+  describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let!(:data) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+
+    it "pool-03 has nodes in correct locations" do
+      expect(data['nodePools']).to include(
+        including(
+          "name" => "pool-03",
+          "locations" => match_array([
+            "us-east4-b",
+            "us-east4-c",
+          ]),
+        )
+      )
     end
   end
 end

From ecc434c9dcbd7af51faa7ceea8274c22eec38406 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 31 Oct 2019 19:15:18 -0500
Subject: [PATCH 58/90] update docs

---
 README.md                                     |  75 -------------
 autogen/README.md                             |   3 +
 .../README.md                                 | 102 +-----------------
 modules/beta-private-cluster/README.md        | 102 +-----------------
 modules/beta-public-cluster/README.md         |  98 +----------------
 .../private-cluster-update-variant/README.md  |  79 --------------
 modules/private-cluster/README.md             |  79 --------------
 7 files changed, 6 insertions(+), 532 deletions(-)

diff --git a/README.md b/README.md
index 15f6aff13b..66623aa0c4 100644
--- a/README.md
+++ b/README.md
@@ -125,81 +125,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| region | Cluster region |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/autogen/README.md b/autogen/README.md
index 846d339911..3efe785ff0 100644
--- a/autogen/README.md
+++ b/autogen/README.md
@@ -55,6 +55,9 @@ module "gke" {
     {
       name               = "default-node-pool"
       machine_type       = "n1-standard-2"
+      {% if beta_cluster %}
+      node_locations     = "us-central1-b,us-central1-c"
+      {% endif %}
       min_count          = 1
       max_count          = 100
       disk_size_gb       = 100
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 93daea71ca..adba7958e7 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -48,6 +48,7 @@ module "gke" {
     {
       name               = "default-node-pool"
       machine_type       = "n1-standard-2"
+      node_locations     = "us-central1-b,us-central1-c"
       min_count          = 1
       max_count          = 100
       disk_size_gb       = 100
@@ -132,107 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
-| region | Cluster region |
-| release\_channel | The release channel of this cluster |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 4096848139..12189ad61d 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -48,6 +48,7 @@ module "gke" {
     {
       name               = "default-node-pool"
       machine_type       = "n1-standard-2"
+      node_locations     = "us-central1-b,us-central1-c"
       min_count          = 1
       max_count          = 100
       disk_size_gb       = 100
@@ -132,107 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
-| region | Cluster region |
-| release\_channel | The release channel of this cluster |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 258ea431e4..87a6b59e08 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -43,6 +43,7 @@ module "gke" {
     {
       name               = "default-node-pool"
       machine_type       = "n1-standard-2"
+      node_locations     = "us-central1-b,us-central1-c"
       min_count          = 1
       max_count          = 100
       disk_size_gb       = 100
@@ -127,103 +128,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
-| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
-| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
-| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
-| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| identity\_namespace | Workload Identity namespace | string | `""` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| istio | (Beta) Enable Istio addon | string | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
-| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| cloudrun\_enabled | Whether CloudRun enabled |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
-| istio\_enabled | Whether Istio is enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
-| region | Cluster region |
-| release\_channel | The release channel of this cluster |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index fa9cdb8852..63c780a2d3 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| region | Cluster region |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 5465544b82..729bca2d46 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
-| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
-| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
-| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
-| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
-| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
-| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
-| description | The description of the cluster | string | `""` | no |
-| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
-| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
-| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
-| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
-| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
-| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
-| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
-| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
-| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
-| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
-| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
-| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
-| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
-| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
-| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
-| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
-| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
-| name | The name of the cluster (required) | string | n/a | yes |
-| network | The VPC network to host the cluster in (required) | string | n/a | yes |
-| network\_policy | Enable network policy addon | bool | `"false"` | no |
-| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
-| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
-| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
-| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
-| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
-| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
-| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
-| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
-| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
-| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
-| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
-| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
-| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
-| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| ca\_certificate | Cluster ca certificate (base64 encoded) |
-| endpoint | Cluster endpoint |
-| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
-| http\_load\_balancing\_enabled | Whether http load balancing enabled |
-| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
-| location | Cluster location (region if regional cluster, zone if zonal cluster) |
-| logging\_service | Logging service used |
-| master\_authorized\_networks\_config | Networks from which access to master is permitted |
-| master\_version | Current master kubernetes version |
-| min\_master\_version | Minimum master kubernetes version |
-| monitoring\_service | Monitoring service used |
-| name | Cluster name |
-| network\_policy\_enabled | Whether network policy enabled |
-| node\_pools\_names | List of node pools names |
-| node\_pools\_versions | List of node pools versions |
-| region | Cluster region |
-| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
-| type | Cluster type (regional / zonal) |
-| zones | List of zones in which the cluster resides |
-
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements

From d87adebe64126763bb458ae62c400beb2e611e21 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 31 Oct 2019 19:18:50 -0500
Subject: [PATCH 59/90] fix docs

---
 README.md                                     |  75 +++++++++++++
 .../README.md                                 | 101 ++++++++++++++++++
 modules/beta-private-cluster/README.md        | 101 ++++++++++++++++++
 modules/beta-public-cluster/README.md         |  97 +++++++++++++++++
 .../private-cluster-update-variant/README.md  |  79 ++++++++++++++
 modules/private-cluster/README.md             |  79 ++++++++++++++
 6 files changed, 532 insertions(+)

diff --git a/README.md b/README.md
index 66623aa0c4..15f6aff13b 100644
--- a/README.md
+++ b/README.md
@@ -125,6 +125,81 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| region | Cluster region |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index adba7958e7..d825838b33 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -133,6 +133,107 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
+| region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 12189ad61d..9f2d8a7ed9 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -133,6 +133,107 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
+| region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 87a6b59e08..7f81e81500 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -128,6 +128,103 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no |
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
+| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no |
+| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| identity\_namespace | Workload Identity namespace | string | `""` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| istio | (Beta) Enable Istio addon | string | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| cloudrun\_enabled | Whether CloudRun enabled |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
+| istio\_enabled | Whether Istio is enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| pod\_security\_policy\_enabled | Whether pod security policy is enabled |
+| region | Cluster region |
+| release\_channel | The release channel of this cluster |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index 63c780a2d3..fa9cdb8852 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| region | Cluster region |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 729bca2d46..5465544b82 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l
 In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
+| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no |
+| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
+| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
+| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
+| description | The description of the cluster | string | `""` | no |
+| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
+| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
+| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
+| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
+| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no |
+| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no |
+| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
+| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no |
+| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no |
+| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
+| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |
+| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no |
+| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `<list>` | no |
+| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no |
+| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no |
+| name | The name of the cluster (required) | string | n/a | yes |
+| network | The VPC network to host the cluster in (required) | string | n/a | yes |
+| network\_policy | Enable network policy addon | bool | `"false"` | no |
+| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no |
+| node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
+| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
+| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
+| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
+| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
+| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
+| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
+| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
+| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
+| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes |
+| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `<list>` | no |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | Cluster ca certificate (base64 encoded) |
+| endpoint | Cluster endpoint |
+| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
+| http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| logging\_service | Logging service used |
+| master\_authorized\_networks\_config | Networks from which access to master is permitted |
+| master\_version | Current master kubernetes version |
+| min\_master\_version | Minimum master kubernetes version |
+| monitoring\_service | Monitoring service used |
+| name | Cluster name |
+| network\_policy\_enabled | Whether network policy enabled |
+| node\_pools\_names | List of node pools names |
+| node\_pools\_versions | List of node pools versions |
+| region | Cluster region |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| type | Cluster type (regional / zonal) |
+| zones | List of zones in which the cluster resides |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 ## Requirements

From 29c2e5978d592e4605e0d6bc8485ce20678f575d Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Thu, 31 Oct 2019 19:30:06 -0500
Subject: [PATCH 60/90] add test to ci

---
 build/int.cloudbuild.yaml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 5a52a5889b..e58a1a66c0 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -41,6 +41,26 @@ steps:
     - verify disable-client-cert-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local']
+- id: create node-pool-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create node-pool-local']
+- id: converge node-pool-local
+  waitFor:
+    - create node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local']
+- id: verify node-pool-local
+  waitFor:
+    - converge node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local']
+- id: destroy node-pool-local
+  waitFor:
+    - verify node-pool-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local']
 - id: create shared-vpc-local
   waitFor:
     - prepare

From 6e3802df7c214a16f2abeae2d7004760400bb6de Mon Sep 17 00:00:00 2001
From: Ken Evensen <kdevensen@google.com>
Date: Fri, 1 Nov 2019 10:14:26 -0700
Subject: [PATCH 61/90] Tying identity namespace output to successful cluster
 creation.

---
 autogen/outputs.tf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/autogen/outputs.tf b/autogen/outputs.tf
index 704569d00e..842502ecea 100644
--- a/autogen/outputs.tf
+++ b/autogen/outputs.tf
@@ -154,4 +154,12 @@ output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
 }
+
+output "identity_namespace" {
+  description = "Workload Identity namespace"
+  value       = var.identity_namespace
+  depends_on = [
+    "google_container_cluster.primary"
+  ]
+}
 {% endif %}

From a79537c8f0a9c7669c84d834b517ab3e9ff32241 Mon Sep 17 00:00:00 2001
From: Ken Evensen <kdevensen@google.com>
Date: Fri, 1 Nov 2019 10:16:46 -0700
Subject: [PATCH 62/90] Post generate

---
 modules/beta-private-cluster-update-variant/README.md  | 1 +
 modules/beta-private-cluster-update-variant/outputs.tf | 8 ++++++++
 modules/beta-private-cluster/README.md                 | 1 +
 modules/beta-private-cluster/outputs.tf                | 8 ++++++++
 modules/beta-public-cluster/README.md                  | 1 +
 modules/beta-public-cluster/outputs.tf                 | 8 ++++++++
 6 files changed, 27 insertions(+)

diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 93daea71ca..e9b13daa59 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -212,6 +212,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| identity\_namespace | Workload Identity namespace |
 | intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
 | istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf
index 956c8c2d5d..fb3f29c401 100644
--- a/modules/beta-private-cluster-update-variant/outputs.tf
+++ b/modules/beta-private-cluster-update-variant/outputs.tf
@@ -153,3 +153,11 @@ output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
 }
+
+output "identity_namespace" {
+  description = "Workload Identity namespace"
+  value       = var.identity_namespace
+  depends_on = [
+    "google_container_cluster.primary"
+  ]
+}
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index 4096848139..ed98fb6093 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -212,6 +212,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| identity\_namespace | Workload Identity namespace |
 | intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
 | istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf
index 956c8c2d5d..fb3f29c401 100644
--- a/modules/beta-private-cluster/outputs.tf
+++ b/modules/beta-private-cluster/outputs.tf
@@ -153,3 +153,11 @@ output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
 }
+
+output "identity_namespace" {
+  description = "Workload Identity namespace"
+  value       = var.identity_namespace
+  depends_on = [
+    "google_container_cluster.primary"
+  ]
+}
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 258ea431e4..a9e71538e2 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -203,6 +203,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | endpoint | Cluster endpoint |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |
 | http\_load\_balancing\_enabled | Whether http load balancing enabled |
+| identity\_namespace | Workload Identity namespace |
 | intranode\_visibility\_enabled | Whether intra-node visibility is enabled |
 | istio\_enabled | Whether Istio is enabled |
 | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled |
diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf
index 956c8c2d5d..fb3f29c401 100644
--- a/modules/beta-public-cluster/outputs.tf
+++ b/modules/beta-public-cluster/outputs.tf
@@ -153,3 +153,11 @@ output "release_channel" {
   description = "The release channel of this cluster"
   value       = var.release_channel
 }
+
+output "identity_namespace" {
+  description = "Workload Identity namespace"
+  value       = var.identity_namespace
+  depends_on = [
+    "google_container_cluster.primary"
+  ]
+}

From 647ffd334202239302e354eb8aea5c1cae6c22b1 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Fri, 1 Nov 2019 12:35:00 +0200
Subject: [PATCH 63/90] Removed `gsutil` credentials fix

---
 build/int.cloudbuild.yaml | 2 +-
 modules/acm/main.tf       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 5a52a5889b..acc848d782 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -130,7 +130,7 @@ steps:
   waitFor:
     - create simple-zonal-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && env > /dev/null && kitchen_do converge simple-zonal-local']
 - id: verify simple-zonal-local
   waitFor:
     - converge simple-zonal-local
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 117e13c94b..10e55e9f88 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" {
 
 resource "null_resource" "acm_operator_config" {
   provisioner "local-exec" {
-    command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
+    command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
   }
 
   provisioner "local-exec" {

From ccaa36c9ed316ed3cf4855cd6771b712d7f43119 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Thu, 31 Oct 2019 02:44:41 +0200
Subject: [PATCH 64/90] Fix lint in simple_regional_privat_beta example

---
 build/int.cloudbuild.yaml                     | 2 ++
 examples/simple_regional_private_beta/main.tf | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index c9c666b238..09b5d3eb7e 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -307,3 +307,5 @@ tags:
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
   _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
+options:
+  machineType: 'N1_HIGHCPU_8'
diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf
index dfc33991fa..4e1d405940 100644
--- a/examples/simple_regional_private_beta/main.tf
+++ b/examples/simple_regional_private_beta/main.tf
@@ -19,8 +19,8 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 2.18.0"
-  region      = var.region
+  version = "~> 2.18.0"
+  region  = var.region
 }
 
 data "google_compute_subnetwork" "subnetwork" {

From f884041ec5c64d21a0ab766ca0a637e649b36ffb Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Mon, 4 Nov 2019 17:12:23 +0200
Subject: [PATCH 65/90] Gsutil creds fix

---
 modules/acm/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 10e55e9f88..117e13c94b 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" {
 
 resource "null_resource" "acm_operator_config" {
   provisioner "local-exec" {
-    command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
+    command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
   }
 
   provisioner "local-exec" {

From 763258511e05e66c859b04634f0a15b640642cc4 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Fri, 8 Nov 2019 11:06:58 -0600
Subject: [PATCH 66/90] fix yaml

---
 .kitchen.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index ff04c88dcf..88ca53374e 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -36,8 +36,8 @@ suites:
       systems:
         - name: disable_client_cert
           backend: local
-            controls:
-              - gcloud
+          controls:
+            - gcloud
   - name: "node_pool"
     driver:
       root_module_directory: test/fixtures/node_pool

From a088b8f24292b8c3fffdfe440dea2d04e477dec4 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Fri, 8 Nov 2019 11:13:36 -0600
Subject: [PATCH 67/90] remove duplicate test suite with latest

---
 .kitchen.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index 88ca53374e..39faa2e1e3 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -38,13 +38,6 @@ suites:
           backend: local
           controls:
             - gcloud
-  - name: "node_pool"
-    driver:
-      root_module_directory: test/fixtures/node_pool
-    verifier:
-      systems:
-        - name: node_pool
-          backend: local
   - name: "shared_vpc"
     driver:
       root_module_directory: test/fixtures/shared_vpc

From a4bfc30169b1b156358751b0ef43882d3f5d4945 Mon Sep 17 00:00:00 2001
From: bharathkkb <bharathkrishnakb@gmail.com>
Date: Fri, 8 Nov 2019 11:19:52 -0600
Subject: [PATCH 68/90] fix cloudbuild

---
 build/int.cloudbuild.yaml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 6a9fcb2d35..09b5d3eb7e 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -41,26 +41,6 @@ steps:
     - verify disable-client-cert-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local']
-- id: create node-pool-local
-  waitFor:
-    - prepare
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create node-pool-local']
-- id: converge node-pool-local
-  waitFor:
-    - create node-pool-local
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local']
-- id: verify node-pool-local
-  waitFor:
-    - converge node-pool-local
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local']
-- id: destroy node-pool-local
-  waitFor:
-    - verify node-pool-local
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local']
 - id: create shared-vpc-local
   waitFor:
     - prepare

From ef536c0c158f0612e3278c6532eecd4b46b8de4f Mon Sep 17 00:00:00 2001
From: "paul.p" <paul.palam@gmail.com>
Date: Fri, 8 Nov 2019 16:41:19 +0200
Subject: [PATCH 69/90] Remove "env" command form build/int.cloudbuild.yaml

Co-Authored-By: Aaron Lane <aarondrewlane@gmail.com>
---
 build/int.cloudbuild.yaml | 2 +-
 modules/acm/main.tf       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 110f1171d4..09b5d3eb7e 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -130,7 +130,7 @@ steps:
   waitFor:
     - create simple-zonal-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && env > /dev/null && kitchen_do converge simple-zonal-local']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local']
 - id: verify simple-zonal-local
   waitFor:
     - converge simple-zonal-local
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 117e13c94b..10e55e9f88 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" {
 
 resource "null_resource" "acm_operator_config" {
   provisioner "local-exec" {
-    command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
+    command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
   }
 
   provisioner "local-exec" {

From 582236723b6b2895835d39e3b5fe77a7950859ed Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Fri, 8 Nov 2019 16:02:21 -0800
Subject: [PATCH 70/90] Fix node_count when autoscaling disabled on node pool.
 Fixes #311 Dont set initial_node_count when autoscaling is disabled on node
 pool. Use new node pool var  when setting desired size of pool - matches
 provider var

---
 autogen/cluster.tf                                     | 10 ++++++----
 cluster.tf                                             | 10 ++++++----
 modules/beta-private-cluster-update-variant/cluster.tf | 10 ++++++----
 modules/beta-private-cluster/cluster.tf                | 10 ++++++----
 modules/beta-public-cluster/cluster.tf                 | 10 ++++++----
 modules/private-cluster-update-variant/cluster.tf      | 10 ++++++----
 modules/private-cluster/cluster.tf                     | 10 ++++++----
 7 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 0bac34a37e..7353468cd7 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -316,16 +316,18 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
   {% if beta_cluster %}
   max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null)
   {% endif %}
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/cluster.tf b/cluster.tf
index 7e5f53ee47..072a60fb14 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -137,13 +137,15 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 6039cecd98..c92695ce49 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -290,14 +290,16 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
   max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null)
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 10e12a9ba0..fb9b1ee683 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -218,14 +218,16 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
   max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null)
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index b5f896bc1b..88725475a8 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -213,14 +213,16 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
   max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null)
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index 615fe84bcc..d7fc2dd736 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -214,13 +214,15 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index 3c42e64325..c8051255bf 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -142,13 +142,15 @@ resource "google_container_node_pool" "pools" {
     "version",
     local.node_version,
   )
-  initial_node_count = lookup(
+
+  initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup(
     var.node_pools[count.index],
     "initial_node_count",
-    lookup(var.node_pools[count.index], "min_count", 1),
-  )
+    lookup(var.node_pools[count.index], "min_count", 1)
+  ) : null
+
 
-  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1)
 
   dynamic "autoscaling" {
     for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []

From 52cd66c49b178a8935e50ee8ae4d498cfe942745 Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Fri, 8 Nov 2019 16:03:09 -0800
Subject: [PATCH 71/90] update test and example to show usage of a node pool
 with autoscaling off

---
 examples/node_pool/main.tf                    | 12 +++
 test/integration/node_pool/controls/gcloud.rb | 97 ++++++++++++++++++-
 .../integration/node_pool/controls/kubectl.rb | 15 +++
 3 files changed, 122 insertions(+), 2 deletions(-)

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index c7a7f852ae..17b3bf8f67 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -59,12 +59,20 @@ module "gke" {
       auto_repair       = false
       service_account   = var.compute_engine_service_account
     },
+    {
+      name              = "pool-03"
+      autoscaling       = false
+      node_count        = 2
+      service_account = var.compute_engine_service_account
+      auto_upgrade    = true
+    },
   ]
 
   node_pools_oauth_scopes = {
     all     = []
     pool-01 = []
     pool-02 = []
+    pool-03 = []
   }
 
   node_pools_metadata = {
@@ -73,6 +81,7 @@ module "gke" {
       shutdown-script = file("${path.module}/data/shutdown-script.sh")
     }
     pool-02 = {}
+    pool-03 = {}
   }
 
   node_pools_labels = {
@@ -83,6 +92,7 @@ module "gke" {
       pool-01-example = true
     }
     pool-02 = {}
+    pool-03 = {}
   }
 
   node_pools_taints = {
@@ -101,6 +111,7 @@ module "gke" {
       },
     ]
     pool-02 = []
+    pool-03 = []
   }
 
   node_pools_tags = {
@@ -111,6 +122,7 @@ module "gke" {
       "pool-01-example",
     ]
     pool-02 = []
+    pool-03 = []
   }
 }
 
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
index 6ff5fdd201..675a2e39ae 100644
--- a/test/integration/node_pool/controls/gcloud.rb
+++ b/test/integration/node_pool/controls/gcloud.rb
@@ -36,8 +36,8 @@
     describe "node pools" do
       let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } }
 
-      it "has 2" do
-        expect(node_pools.count).to eq 2
+      it "has 3" do
+        expect(node_pools.count).to eq 3
       end
 
       describe "pool-01" do
@@ -279,6 +279,99 @@
           )
         end
       end
+
+      describe "pool-03" do
+        it "exists" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+            )
+          )
+        end
+
+        it "is the expected machine type" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "machineType" => "n1-standard-2",
+              ),
+            )
+          )
+        end
+
+        it "has autoscaling disabled" do
+          expect(data['nodePools']).not_to include(
+            including(
+              "name" => "pool-03",
+              "autoscaling" => including(
+                "enabled" => true,
+              ),
+            )
+          )
+        end
+
+        it "has the expected node count" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "initialNodeCount" => 2
+            )
+          )
+        end
+
+        it "has autorepair enabled" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "management" => including(
+                "autoRepair" => true,
+              ),
+            )
+          )
+        end
+
+        it "has automatic upgrades enabled" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "management" => including(
+                "autoUpgrade" => true,
+              ),
+            )
+          )
+        end
+
+        it "has the expected labels" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "labels" => {
+                  "all-pools-example" => "true",
+                  "cluster_name" => cluster_name,
+                  "node_pool" => "pool-03",
+                },
+              ),
+            )
+          )
+        end
+
+        it "has the expected network tags" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "tags" => match_array([
+                  "all-node-example",
+                  "gke-#{cluster_name}",
+                  "gke-#{cluster_name}-pool-03",
+                ]),
+              ),
+            )
+          )
+        end
+      end
     end
   end
 end
diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb
index 471f9cb33f..811ebcda0f 100644
--- a/test/integration/node_pool/controls/kubectl.rb
+++ b/test/integration/node_pool/controls/kubectl.rb
@@ -72,6 +72,21 @@
           all_nodes.select { |n| n.metadata.labels.node_pool == "pool-02" }
         end
 
+        it "has the expected taints" do
+          expect(taints).to include(
+            {
+              effect: "PreferNoSchedule",
+              key: "all-pools-example",
+              value: "true",
+            }
+          )
+        end
+      end
+      describe "pool-03" do
+        let(:nodes) do
+          all_nodes.select { |n| n.metadata.labels.node_pool == "pool-03" }
+        end
+
         it "has the expected taints" do
           expect(taints).to include(
             {

From e82a77e45a24a27d966ace400239d7b7bc64766e Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Fri, 8 Nov 2019 16:21:26 -0800
Subject: [PATCH 72/90] fix formatting

---
 examples/node_pool/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index 17b3bf8f67..bf892be2a9 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -60,9 +60,9 @@ module "gke" {
       service_account   = var.compute_engine_service_account
     },
     {
-      name              = "pool-03"
-      autoscaling       = false
-      node_count        = 2
+      name            = "pool-03"
+      autoscaling     = false
+      node_count      = 2
       service_account = var.compute_engine_service_account
       auto_upgrade    = true
     },

From 70688eeb611e4371030ee61b9d9115f0f0f3a21e Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Mon, 11 Nov 2019 11:23:52 -0800
Subject: [PATCH 73/90] force codebuild


From ee90e456da14afb82e7147f50f789239a8468e03 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Mon, 11 Nov 2019 15:12:56 -0500
Subject: [PATCH 74/90] Update CHANGELOG.md

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 74a293e683..ba31673202 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 ### Added
 
 * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300]
+* Support for setting node_locations on node pools. [#303]
 
 ## [v5.1.1] - 2019-10-25
 
@@ -225,6 +226,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
+[#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303
 [#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286
 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285
 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284

From 48b94e8ff4f586ff113593e107fab328da536870 Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Tue, 12 Nov 2019 14:33:23 -0800
Subject: [PATCH 75/90] add to changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba31673202..226520c116 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300]
 * Support for setting node_locations on node pools. [#303]
+* Fix for specifying  `node_count` on node pools when autoscaling is disabled. [#313]
 
 ## [v5.1.1] - 2019-10-25
 

From e3ae31ae1289754bc71916cbafacddd627885161 Mon Sep 17 00:00:00 2001
From: Taylor Ludwig <taylorludwig@gmail.com>
Date: Tue, 12 Nov 2019 14:34:28 -0800
Subject: [PATCH 76/90] move node_pool example to us-central1 where there is
 cpu quota available after merging with master us-east4 maxes out default
 quota of 72 cpus in the region

---
 examples/node_pool/main.tf                    | 2 +-
 test/fixtures/shared/variables.tf             | 5 ++---
 test/integration/node_pool/controls/gcloud.rb | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index ad8d076f98..5bc0f53407 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -60,7 +60,7 @@ module "gke" {
     },
     {
       name            = "pool-03"
-      node_locations  = "us-east4-b,us-east4-c"
+      node_locations  = "${var.region}-b,${var.region}-c"
       autoscaling     = false
       node_count      = 2
       machine_type    = "n1-standard-2"
diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf
index 5dff24dbd4..9760d65a94 100644
--- a/test/fixtures/shared/variables.tf
+++ b/test/fixtures/shared/variables.tf
@@ -20,13 +20,13 @@ variable "project_id" {
 
 variable "region" {
   description = "The GCP region to create and test resources in"
-  default     = "us-east4"
+  default     = "us-central1"
 }
 
 variable "zones" {
   type        = list(string)
   description = "The GCP zones to create and test resources in, for applicable tests"
-  default     = ["us-east4-a", "us-east4-b", "us-east4-c"]
+  default     = ["us-central1-a", "us-central1-b", "us-central1-c"]
 }
 
 variable "compute_engine_service_account" {
@@ -36,4 +36,3 @@ variable "compute_engine_service_account" {
 variable "registry_project_id" {
   description = "Project to use for granting access to the GCR registry, if requested"
 }
-
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
index 2c6ed4f648..69a15e8293 100644
--- a/test/integration/node_pool/controls/gcloud.rb
+++ b/test/integration/node_pool/controls/gcloud.rb
@@ -392,8 +392,8 @@
         including(
           "name" => "pool-03",
           "locations" => match_array([
-            "us-east4-b",
-            "us-east4-c",
+            "us-central1-b",
+            "us-central1-c",
           ]),
         )
       )

From edb49636f31e78e195f9fdd4c3ac577a1dea6bb0 Mon Sep 17 00:00:00 2001
From: pp <paul.palam@gmail.com>
Date: Wed, 13 Nov 2019 14:27:47 +0200
Subject: [PATCH 77/90] Updated docker image ver to fix gsutil cerds issue

---
 build/int.cloudbuild.yaml  | 2 +-
 build/lint.cloudbuild.yaml | 2 +-
 test/setup/iam.tf          | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 09b5d3eb7e..086203864d 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -306,6 +306,6 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4'
 options:
   machineType: 'N1_HIGHCPU_8'
diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml
index 7ba0827bdb..02b9e5327b 100644
--- a/build/lint.cloudbuild.yaml
+++ b/build/lint.cloudbuild.yaml
@@ -24,4 +24,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4'
diff --git a/test/setup/iam.tf b/test/setup/iam.tf
index fd8a62a18f..7ff4de74bc 100644
--- a/test/setup/iam.tf
+++ b/test/setup/iam.tf
@@ -25,7 +25,8 @@ locals {
     "roles/iam.serviceAccountUser",
     "roles/compute.networkAdmin",
     "roles/compute.viewer",
-    "roles/resourcemanager.projectIamAdmin"
+    "roles/resourcemanager.projectIamAdmin",
+    "roles/composer.worker"
   ]
 }
 

From 67bd2c8436ae59995225ba5a4f870034cce1f4bf Mon Sep 17 00:00:00 2001
From: Aaron Lane <aaronlane@google.com>
Date: Wed, 13 Nov 2019 11:09:21 -0500
Subject: [PATCH 78/90] Add links for #300, #311 to CHANGELOG

---
 CHANGELOG.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 226520c116..e839926e6b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,7 +12,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 
 * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300]
 * Support for setting node_locations on node pools. [#303]
-* Fix for specifying  `node_count` on node pools when autoscaling is disabled. [#313]
+* Fix for specifying  `node_count` on node pools when autoscaling is disabled. [#311]
 
 ## [v5.1.1] - 2019-10-25
 
@@ -227,7 +227,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
+[#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311
 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303
+[#300]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/300
 [#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286
 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285
 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284

From e7851baeda5ba65167ddd960ba4cfdbdfec5bad5 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 15:27:10 -0500
Subject: [PATCH 79/90] Rename simple_zonal_with_acm example

---
 examples/{simple_zonal => simple_zonal_with_acm}/README.md  | 6 ++++--
 examples/{simple_zonal => simple_zonal_with_acm}/acm.tf     | 0
 examples/{simple_zonal => simple_zonal_with_acm}/main.tf    | 0
 examples/{simple_zonal => simple_zonal_with_acm}/outputs.tf | 0
 .../{simple_zonal => simple_zonal_with_acm}/test_outputs.tf | 0
 .../{simple_zonal => simple_zonal_with_acm}/variables.tf    | 3 +--
 test/fixtures/simple_zonal/example.tf                       | 3 +--
 test/fixtures/simple_zonal/network.tf                       | 1 -
 8 files changed, 6 insertions(+), 7 deletions(-)
 rename examples/{simple_zonal => simple_zonal_with_acm}/README.md (85%)
 rename examples/{simple_zonal => simple_zonal_with_acm}/acm.tf (100%)
 rename examples/{simple_zonal => simple_zonal_with_acm}/main.tf (100%)
 rename examples/{simple_zonal => simple_zonal_with_acm}/outputs.tf (100%)
 rename examples/{simple_zonal => simple_zonal_with_acm}/test_outputs.tf (100%)
 rename examples/{simple_zonal => simple_zonal_with_acm}/variables.tf (95%)

diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal_with_acm/README.md
similarity index 85%
rename from examples/simple_zonal/README.md
rename to examples/simple_zonal_with_acm/README.md
index 2e514c6618..ea8539dc5f 100644
--- a/examples/simple_zonal/README.md
+++ b/examples/simple_zonal_with_acm/README.md
@@ -1,13 +1,15 @@
 # Simple Zonal Cluster
 
-This example illustrates how to create a simple cluster.
+This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/).
+
+It incorporates the standard cluster module and the [ACM install module](../../modules/acm).
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| acm\_policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | `"foo-corp"` | no |
+| acm\_policy\_dir | Subfolder containing configs in ACM Git repo | string | `"foo-corp"` | no |
 | acm\_sync\_branch | Anthos config management Git branch | string | `"1.0.0"` | no |
 | acm\_sync\_repo | Anthos config management Git repo | string | `"git@github.com:GoogleCloudPlatform/csp-config-management.git"` | no |
 | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal_with_acm/acm.tf
similarity index 100%
rename from examples/simple_zonal/acm.tf
rename to examples/simple_zonal_with_acm/acm.tf
diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal_with_acm/main.tf
similarity index 100%
rename from examples/simple_zonal/main.tf
rename to examples/simple_zonal_with_acm/main.tf
diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal_with_acm/outputs.tf
similarity index 100%
rename from examples/simple_zonal/outputs.tf
rename to examples/simple_zonal_with_acm/outputs.tf
diff --git a/examples/simple_zonal/test_outputs.tf b/examples/simple_zonal_with_acm/test_outputs.tf
similarity index 100%
rename from examples/simple_zonal/test_outputs.tf
rename to examples/simple_zonal_with_acm/test_outputs.tf
diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal_with_acm/variables.tf
similarity index 95%
rename from examples/simple_zonal/variables.tf
rename to examples/simple_zonal_with_acm/variables.tf
index 39a97b02d0..2f427af33b 100644
--- a/examples/simple_zonal/variables.tf
+++ b/examples/simple_zonal_with_acm/variables.tf
@@ -61,8 +61,7 @@ variable "acm_sync_branch" {
 }
 
 variable "acm_policy_dir" {
-  description = "Subfolder containing configs in Ahtons config management Git repo"
+  description = "Subfolder containing configs in ACM Git repo"
   type        = string
   default     = "foo-corp"
 }
-
diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf
index c0edb35e2b..4b76479dc5 100644
--- a/test/fixtures/simple_zonal/example.tf
+++ b/test/fixtures/simple_zonal/example.tf
@@ -15,7 +15,7 @@
  */
 
 module "example" {
-  source = "../../../examples/simple_zonal"
+  source = "../../../examples/simple_zonal_with_acm"
 
   project_id          = var.project_id
   cluster_name_suffix = "-${random_string.suffix.result}"
@@ -26,4 +26,3 @@ module "example" {
   ip_range_pods       = google_compute_subnetwork.main.secondary_ip_range[0].range_name
   ip_range_services   = google_compute_subnetwork.main.secondary_ip_range[1].range_name
 }
-
diff --git a/test/fixtures/simple_zonal/network.tf b/test/fixtures/simple_zonal/network.tf
index e1292eae3b..a4978c9ac3 100644
--- a/test/fixtures/simple_zonal/network.tf
+++ b/test/fixtures/simple_zonal/network.tf
@@ -45,4 +45,3 @@ resource "google_compute_subnetwork" "main" {
     ip_cidr_range = "192.168.64.0/18"
   }
 }
-

From 685b52593cfb04c3f373ef8bcc9d698141580bc3 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 15:56:55 -0500
Subject: [PATCH 80/90] Work on ACM docs

---
 modules/acm/README.md | 30 +++++++++++++++++++++++++++---
 modules/acm/main.tf   |  8 ++++----
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/modules/acm/README.md b/modules/acm/README.md
index 8bc8abc9f0..7691fd6046 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -1,10 +1,34 @@
 # Terraform Kubernetes Engine ACM Submodule
 
-This module installs Anthos Config Management (ACM) in a Kubernetes cluster.
-To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/).
+This module installs [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/) (ACM) in a Kubernetes cluster.
 
+Specifically, this module automates the following steps for [installing ACM](https://cloud.google.com/anthos-config-management/docs/how-to/installing):
+1. Installing the ACM Operator on your cluster.
+2. Generating an SSH key for accessing Git and providing it to the Operator
+3. Configuring the Operator to connect to your ACM repository
 
-## Configure a Service Account
+## Usage
+
+There is a [full example](../../examples/simple_zonal) provided. Simple usage is as follows:
+
+```tf
+module "acm" {
+  source           = "../../modules/acm"
+  project_id       = var.project_id
+  location         = module.gke.location
+  cluster_name     = module.gke.name
+  sync_repo        = var.acm_sync_repo
+  sync_branch      = var.acm_sync_branch
+  policy_dir       = var.acm_policy_dir
+  cluster_endpoint = module.gke.endpoint
+}
+```
+
+
+In addition to this [example](../../examples/simple_zonal) shows how to provision a cluster and install ACm. 
+
+
+In order to use this module, you must use a Service Account 
 
 In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md)
 plus **roles/container.admin** role.
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 10e55e9f88..1216c27e0f 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -40,7 +40,7 @@ resource "null_resource" "acm_operator_config" {
   }
 
   provisioner "local-exec" {
-    when    = "destroy"
+    when    = destroy
     command = "rm -f ${path.module}/config-management-operator.yaml"
   }
 }
@@ -51,7 +51,7 @@ resource "null_resource" "acm_operator" {
   }
 
   provisioner "local-exec" {
-    when    = "destroy"
+    when    = destroy
     command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml"
   }
 
@@ -68,7 +68,7 @@ resource "null_resource" "git_creds_secret" {
   }
 
   provisioner "local-exec" {
-    when    = "destroy"
+    when    = destroy
     command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete secret git-creds -n=config-management-system"
   }
 
@@ -95,7 +95,7 @@ resource "null_resource" "acm_config" {
   }
 
   provisioner "local-exec" {
-    when    = "destroy"
+    when    = destroy
     command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f -"
   }
 

From 018c6fb93dc90065e828cc066eedb7a35bd0fec1 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 16:55:39 -0500
Subject: [PATCH 81/90] Make private clusters expose public endpoint if
 var.deploy_using_private_endpoint is false

---
 autogen/main.tf                                     | 2 +-
 modules/beta-private-cluster-update-variant/main.tf | 2 +-
 modules/beta-private-cluster/main.tf                | 2 +-
 modules/private-cluster-update-variant/main.tf      | 2 +-
 modules/private-cluster/main.tf                     | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/autogen/main.tf b/autogen/main.tf
index 30347b9b15..841444ea44 100644
--- a/autogen/main.tf
+++ b/autogen/main.tf
@@ -92,7 +92,7 @@ locals {
   cluster_output_zones          = local.cluster_output_regional_zones
 
 {% if private_cluster %}
-  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint
+  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint
 {% else %}
   cluster_output_endpoint = google_container_cluster.primary.endpoint
 {% endif %}
diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
index 2de95c063d..5b235ce00f 100644
--- a/modules/beta-private-cluster-update-variant/main.tf
+++ b/modules/beta-private-cluster-update-variant/main.tf
@@ -83,7 +83,7 @@ locals {
   cluster_output_zonal_zones    = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : []
   cluster_output_zones          = local.cluster_output_regional_zones
 
-  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint
+  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint
 
   cluster_output_master_auth                        = concat(google_container_cluster.primary.*.master_auth, [])
   cluster_output_master_version                     = google_container_cluster.primary.master_version
diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
index 2de95c063d..5b235ce00f 100644
--- a/modules/beta-private-cluster/main.tf
+++ b/modules/beta-private-cluster/main.tf
@@ -83,7 +83,7 @@ locals {
   cluster_output_zonal_zones    = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : []
   cluster_output_zones          = local.cluster_output_regional_zones
 
-  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint
+  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint
 
   cluster_output_master_auth                        = concat(google_container_cluster.primary.*.master_auth, [])
   cluster_output_master_version                     = google_container_cluster.primary.master_version
diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
index aba5e2d79f..f0d307311c 100644
--- a/modules/private-cluster-update-variant/main.tf
+++ b/modules/private-cluster-update-variant/main.tf
@@ -70,7 +70,7 @@ locals {
   cluster_output_zonal_zones    = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : []
   cluster_output_zones          = local.cluster_output_regional_zones
 
-  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint
+  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint
 
   cluster_output_master_auth                        = concat(google_container_cluster.primary.*.master_auth, [])
   cluster_output_master_version                     = google_container_cluster.primary.master_version
diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
index aba5e2d79f..f0d307311c 100644
--- a/modules/private-cluster/main.tf
+++ b/modules/private-cluster/main.tf
@@ -70,7 +70,7 @@ locals {
   cluster_output_zonal_zones    = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : []
   cluster_output_zones          = local.cluster_output_regional_zones
 
-  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint
+  cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint
 
   cluster_output_master_auth                        = concat(google_container_cluster.primary.*.master_auth, [])
   cluster_output_master_version                     = google_container_cluster.primary.master_version

From b5ebae53357b29a171f8136796a5c152166c59e9 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 17:46:26 -0500
Subject: [PATCH 82/90] Make key generation optional for ACM module

---
 modules/acm/main.tf      | 12 ++++++++++--
 modules/acm/outputs.tf   |  2 +-
 modules/acm/variables.tf |  5 +++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 1216c27e0f..439cb02fda 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -18,6 +18,7 @@ locals {
   cluster_endpoint       = "https://${var.cluster_endpoint}"
   token                  = data.google_client_config.default.access_token
   cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate
+  private_key            = var.create_ssh_key ? tls_private_key.git_creds[0].private_key_pem : ""
 }
 
 data "google_container_cluster" "primary" {
@@ -30,6 +31,7 @@ data "google_client_config" "default" {
 }
 
 resource "tls_private_key" "git_creds" {
+  count     = var.create_ssh_key ? 1 : 0
   algorithm = "RSA"
   rsa_bits  = 4096
 }
@@ -63,8 +65,10 @@ resource "null_resource" "acm_operator" {
 }
 
 resource "null_resource" "git_creds_secret" {
+  count = var.create_ssh_key ? 1 : 0
+
   provisioner "local-exec" {
-    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${tls_private_key.git_creds.private_key_pem}'"
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${local.private_key}'"
   }
 
   provisioner "local-exec" {
@@ -85,11 +89,15 @@ data "template_file" "acm_config" {
     sync_repo    = var.sync_repo
     sync_branch  = var.sync_branch
     policy_dir   = var.policy_dir
-    secret_type  = "ssh"
+    secret_type  = var.create_ssh_key ? "ssh" : "none"
   }
 }
 
 resource "null_resource" "acm_config" {
+  triggers = {
+    config = data.template_file.acm_config.rendered
+  }
+
   provisioner "local-exec" {
     command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f -"
   }
diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf
index 0ebca72e42..1945ba4f59 100644
--- a/modules/acm/outputs.tf
+++ b/modules/acm/outputs.tf
@@ -16,6 +16,6 @@
 
 output "git_creds_public" {
   description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository."
-  value       = tls_private_key.git_creds.public_key_openssh
+  value       = var.create_ssh_key ? tls_private_key.git_creds.*.public_key_openssh : null
 }
 
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
index 94b720e479..faa91b201a 100644
--- a/modules/acm/variables.tf
+++ b/modules/acm/variables.tf
@@ -50,3 +50,8 @@ variable "cluster_endpoint" {
   type        = string
 }
 
+variable "create_ssh_key" {
+  description = "Controls whether a key will be generated for Git authentication"
+  type        = bool
+  default     = true
+}

From 0ee7ca4966d98f6ef87f16b2c116f02d68583eec Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 17:53:47 -0500
Subject: [PATCH 83/90] Clean up ACM variable descriptions

---
 modules/acm/README.md    | 11 ++++++-----
 modules/acm/variables.tf | 10 +++++-----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/modules/acm/README.md b/modules/acm/README.md
index 7691fd6046..9beb6ee270 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -48,12 +48,13 @@ To enable Git access to the configuration repository over SSH, complete step 2 i
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes |
-| cluster\_name | The name of the cluster. | string | n/a | yes |
-| location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes |
-| policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes |
+| cluster\_name | The unique name to identify the cluster by in ACM. | string | n/a | yes |
+| create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no |
+| location | The location (zone or region) this cluster has been created in. | string | n/a | yes |
+| policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes |
 | project\_id | The project in which the resource belongs. | string | n/a | yes |
-| sync\_branch | Anthos config management Git branch | string | `"master"` | no |
-| sync\_repo | Anthos config management Git repo | string | n/a | yes |
+| sync\_branch | ACM repo Git branch | string | `"master"` | no |
+| sync\_repo | ACM Git repo address | string | n/a | yes |
 
 ## Outputs
 
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
index faa91b201a..164e14d846 100644
--- a/modules/acm/variables.tf
+++ b/modules/acm/variables.tf
@@ -15,7 +15,7 @@
  */
 
 variable "cluster_name" {
-  description = "The name of the cluster."
+  description = "The unique name to identify the cluster in ACM."
   type        = string
 }
 
@@ -25,23 +25,23 @@ variable "project_id" {
 }
 
 variable "location" {
-  description = "The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified."
+  description = "The location (zone or region) this cluster has been created in."
   type        = string
 }
 
 variable "sync_repo" {
-  description = "Anthos config management Git repo"
+  description = "ACM Git repo address"
   type        = string
 }
 
 variable "sync_branch" {
-  description = "Anthos config management Git branch"
+  description = "ACM repo Git branch"
   type        = string
   default     = "master"
 }
 
 variable "policy_dir" {
-  description = "Subfolder containing configs in Ahtons config management Git repo"
+  description = "Subfolder containing configs in ACM Git repo"
   type        = string
 }
 

From 8d174c6cc1974349ab434ef9a92d34749f2fe129 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 18:23:28 -0500
Subject: [PATCH 84/90] Add test for InSpec operator

---
 .kitchen.yml                                  |  1 +
 test/integration/simple_zonal/controls/acm.rb | 52 +++++++++++++++++++
 test/integration/simple_zonal/inspec.yml      |  4 +-
 3 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 test/integration/simple_zonal/controls/acm.rb

diff --git a/.kitchen.yml b/.kitchen.yml
index 39faa2e1e3..c0d08c2418 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -85,6 +85,7 @@ suites:
           backend: local
           controls:
             - gcloud
+            - acm
         - name: gcp
           backend: gcp
           controls:
diff --git a/test/integration/simple_zonal/controls/acm.rb b/test/integration/simple_zonal/controls/acm.rb
new file mode 100644
index 0000000000..45663f176d
--- /dev/null
+++ b/test/integration/simple_zonal/controls/acm.rb
@@ -0,0 +1,52 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'kubeclient'
+require 'rest-client'
+
+require 'base64'
+
+kubernetes_endpoint = attribute('kubernetes_endpoint')
+client_token = attribute('client_token')
+ca_certificate = attribute('ca_certificate')
+
+control "acm" do
+  title "Anthos Config Management"
+
+  describe "kubernetes" do
+    let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" }
+    let(:client) do
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate)))
+      Kubeclient::Client.new(
+        kubernetes_http_endpoint,
+        "v1",
+        ssl_options: {
+          cert_store: cert_store,
+          verify_ssl: OpenSSL::SSL::VERIFY_PEER,
+        },
+        auth_options: {
+          bearer_token: Base64.decode64(client_token),
+        },
+      )
+    end
+
+    describe "config-management-system namespace" do
+      let(:namespace) { client.get_namespace("config-management-system") }
+      it "should exist" do
+        expect(namespace).not_to be nil
+      end
+    end
+  end
+end
diff --git a/test/integration/simple_zonal/inspec.yml b/test/integration/simple_zonal/inspec.yml
index 3a3bc74145..4e41f27bad 100644
--- a/test/integration/simple_zonal/inspec.yml
+++ b/test/integration/simple_zonal/inspec.yml
@@ -25,4 +25,6 @@ attributes:
   - name: service_account
     required: true
     type: string
-
+  - name: ca_certificate
+    required: true
+    type: string

From 80b5704986510726d78439752ca37e971dc70328 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 19:15:18 -0500
Subject: [PATCH 85/90] Work on downloading operator in CI

---
 build/int.cloudbuild.yaml                   |  3 ++
 examples/simple_zonal_with_acm/acm.tf       |  1 +
 examples/simple_zonal_with_acm/variables.tf |  6 ++++
 modules/acm/README.md                       | 38 +++++++++++----------
 modules/acm/main.tf                         |  8 +++--
 modules/acm/variables.tf                    |  6 ++++
 test/fixtures/simple_zonal/example.tf       |  1 +
 test/task_helper_functions.sh               |  5 +++
 8 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
index 086203864d..e78d0eb2ba 100644
--- a/build/int.cloudbuild.yaml
+++ b/build/int.cloudbuild.yaml
@@ -14,6 +14,9 @@
 
 timeout: 12600s
 steps:
+- id: download acm
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && download_acm']
 - id: prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment']
diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf
index 47fdce1d86..0af8f08b5d 100644
--- a/examples/simple_zonal_with_acm/acm.tf
+++ b/examples/simple_zonal_with_acm/acm.tf
@@ -23,4 +23,5 @@ module "acm" {
   sync_branch      = var.acm_sync_branch
   policy_dir       = var.acm_policy_dir
   cluster_endpoint = module.gke.endpoint
+  operator_path    = var.operator_path
 }
diff --git a/examples/simple_zonal_with_acm/variables.tf b/examples/simple_zonal_with_acm/variables.tf
index 2f427af33b..4d7fce1d3a 100644
--- a/examples/simple_zonal_with_acm/variables.tf
+++ b/examples/simple_zonal_with_acm/variables.tf
@@ -65,3 +65,9 @@ variable "acm_policy_dir" {
   type        = string
   default     = "foo-corp"
 }
+
+variable "operator_path" {
+  description = "Path to the operator yaml config. If unset, will download from GCS releases."
+  type        = string
+  default     = null
+}
diff --git a/modules/acm/README.md b/modules/acm/README.md
index 9beb6ee270..1dabfc2ddb 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -13,29 +13,31 @@ There is a [full example](../../examples/simple_zonal) provided. Simple usage is
 
 ```tf
 module "acm" {
-  source           = "../../modules/acm"
-  project_id       = var.project_id
+  source           = "terraform-google-modules/kubernetes-engine/google//modules/acm"
+
+  project_id       = "my-project-id"
+  cluster_name     = "my-cluster-name"
   location         = module.gke.location
-  cluster_name     = module.gke.name
-  sync_repo        = var.acm_sync_repo
-  sync_branch      = var.acm_sync_branch
-  policy_dir       = var.acm_policy_dir
   cluster_endpoint = module.gke.endpoint
+
+  sync_repo        = "git@github.com:GoogleCloudPlatform/csp-config-management.git"
+  sync_branch      = "1.0.0"
+  policy_dir       = "foo-corp"
 }
 ```
 
-
-In addition to this [example](../../examples/simple_zonal) shows how to provision a cluster and install ACm. 
-
-
-In order to use this module, you must use a Service Account 
-
-In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md)
-plus **roles/container.admin** role.
-
-## Usage example
-
-See [examples/simple_zonal](../../examples/simple_zonal) cluster example.
+To deploy this config:
+1. Run `terraform apply`
+2. Inspect the `git\_creds\_public` [output][#outputs] to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience:
+
+   * [{{source_repos_name}}](/source-repositories/docs/authentication#ssh)
+   * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html){:.external}
+   * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/){:.external}.
+   * [Gitlab](https://docs.gitlab.com/ee/ssh/){:.external}
+ 
+## Whitelisting
+Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license.
+By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location.
 
 ## Installation
 
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index 439cb02fda..d93dbdcdf3 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -19,6 +19,8 @@ locals {
   token                  = data.google_client_config.default.access_token
   cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate
   private_key            = var.create_ssh_key ? tls_private_key.git_creds[0].private_key_pem : ""
+  download_operator      = var.operator_path == null ? true : false
+  operator_path          = local.download_operator ? "${path.module}/config-management-operator.yaml" : var.operator_path
 }
 
 data "google_container_cluster" "primary" {
@@ -37,6 +39,8 @@ resource "tls_private_key" "git_creds" {
 }
 
 resource "null_resource" "acm_operator_config" {
+  count = local.download_operator ? 1 : 0
+
   provisioner "local-exec" {
     command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml"
   }
@@ -49,12 +53,12 @@ resource "null_resource" "acm_operator_config" {
 
 resource "null_resource" "acm_operator" {
   provisioner "local-exec" {
-    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${path.module}/config-management-operator.yaml"
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${local.operator_path}"
   }
 
   provisioner "local-exec" {
     when    = destroy
-    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml"
+    command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${local.operator_path}"
   }
 
   depends_on = [
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
index 164e14d846..cc49006ae9 100644
--- a/modules/acm/variables.tf
+++ b/modules/acm/variables.tf
@@ -29,6 +29,12 @@ variable "location" {
   type        = string
 }
 
+variable "operator_path" {
+  description = "Path to the operator yaml config. If unset, will download from GCS releases."
+  type        = string
+  default     = null
+}
+
 variable "sync_repo" {
   description = "ACM Git repo address"
   type        = string
diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf
index 4b76479dc5..7d558e36f0 100644
--- a/test/fixtures/simple_zonal/example.tf
+++ b/test/fixtures/simple_zonal/example.tf
@@ -25,4 +25,5 @@ module "example" {
   subnetwork          = google_compute_subnetwork.main.name
   ip_range_pods       = google_compute_subnetwork.main.secondary_ip_range[0].range_name
   ip_range_services   = google_compute_subnetwork.main.secondary_ip_range[1].range_name
+  operator_path       = "/workspace/acm.yaml"
 }
diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh
index 7de28bbdfe..a4bac86bf6 100755
--- a/test/task_helper_functions.sh
+++ b/test/task_helper_functions.sh
@@ -50,3 +50,8 @@ function check_generate() {
   rm -Rf "${tempdir}"
   return $((rval))
 }
+
+# Pre-download the Anthos Config Management operator
+function download_acm() {
+   gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml /workspace/acm.yaml
+}

From 7e4fe424bba89ec51d4df0e3f336e126f358d130 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 19:24:16 -0500
Subject: [PATCH 86/90] ACM docs cleanup

---
 examples/simple_zonal_with_acm/README.md |  1 +
 modules/acm/README.md                    | 18 +++++++-----------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md
index ea8539dc5f..b5def60e96 100644
--- a/examples/simple_zonal_with_acm/README.md
+++ b/examples/simple_zonal_with_acm/README.md
@@ -16,6 +16,7 @@ It incorporates the standard cluster module and the [ACM install module](../../m
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
 | network | The VPC network to host the cluster in | string | n/a | yes |
+| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
 | subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
diff --git a/modules/acm/README.md b/modules/acm/README.md
index 1dabfc2ddb..bb05629898 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -28,31 +28,27 @@ module "acm" {
 
 To deploy this config:
 1. Run `terraform apply`
-2. Inspect the `git\_creds\_public` [output][#outputs] to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience:
+2. Inspect the `git_creds_public` [output](#outputs) to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience:
 
-   * [{{source_repos_name}}](/source-repositories/docs/authentication#ssh)
-   * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html){:.external}
-   * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/){:.external}.
-   * [Gitlab](https://docs.gitlab.com/ee/ssh/){:.external}
+   * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh)
+   * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html)
+   * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/)
+   * [Gitlab](https://docs.gitlab.com/ee/ssh/)
  
 ## Whitelisting
 Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license.
 By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location.
 
-## Installation
-
-This module automates the instructions described in the [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) guide.
-To enable Git access to the configuration repository over SSH, complete step 2 in the [Using an SSH keypair](https://cloud.google.com/anthos-config-management/docs/how-to/installing#git-creds-ssh) section using the SSH public key from the **git\_creds\_public** output.
-
  <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes |
-| cluster\_name | The unique name to identify the cluster by in ACM. | string | n/a | yes |
+| cluster\_name | The unique name to identify the cluster in ACM. | string | n/a | yes |
 | create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no |
 | location | The location (zone or region) this cluster has been created in. | string | n/a | yes |
+| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no |
 | policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes |
 | project\_id | The project in which the resource belongs. | string | n/a | yes |
 | sync\_branch | ACM repo Git branch | string | `"master"` | no |

From f8d02c199e09e744ed1fa666f56bff1c8326f8ad Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Thu, 14 Nov 2019 19:29:18 -0500
Subject: [PATCH 87/90] Add Policy Controller options to ACM installer

---
 modules/acm/README.md                    | 13 +++++++------
 modules/acm/main.tf                      | 12 +++++++-----
 modules/acm/templates/acm-config.yml.tpl |  4 +++-
 modules/acm/variables.tf                 | 12 ++++++++++++
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/modules/acm/README.md b/modules/acm/README.md
index bb05629898..166611e0b7 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -30,11 +30,11 @@ To deploy this config:
 1. Run `terraform apply`
 2. Inspect the `git_creds_public` [output](#outputs) to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience:
 
-   * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh)
-   * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html)
-   * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/)
-   * [Gitlab](https://docs.gitlab.com/ee/ssh/)
- 
+  * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh)
+  * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html)
+  * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/)
+  * [Gitlab](https://docs.gitlab.com/ee/ssh/)
+
 ## Whitelisting
 Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license.
 By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location.
@@ -47,6 +47,8 @@ By default, this module will attempt to download the ACM operator from Google di
 | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes |
 | cluster\_name | The unique name to identify the cluster in ACM. | string | n/a | yes |
 | create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no |
+| enable\_policy\_controller | Whether to enable the ACM Policy Controller on the cluster | bool | `"true"` | no |
+| install\_template\_library | Whether to install the default Policy Controller template library | bool | `"true"` | no |
 | location | The location (zone or region) this cluster has been created in. | string | n/a | yes |
 | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no |
 | policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes |
@@ -61,4 +63,3 @@ By default, this module will attempt to download the ACM operator from Google di
 | git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. |
 
  <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
-
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
index d93dbdcdf3..3a78172ad6 100644
--- a/modules/acm/main.tf
+++ b/modules/acm/main.tf
@@ -89,11 +89,13 @@ data "template_file" "acm_config" {
   template = file("${path.module}/templates/acm-config.yml.tpl")
 
   vars = {
-    cluster_name = var.cluster_name
-    sync_repo    = var.sync_repo
-    sync_branch  = var.sync_branch
-    policy_dir   = var.policy_dir
-    secret_type  = var.create_ssh_key ? "ssh" : "none"
+    cluster_name             = var.cluster_name
+    sync_repo                = var.sync_repo
+    sync_branch              = var.sync_branch
+    policy_dir               = var.policy_dir
+    secret_type              = var.create_ssh_key ? "ssh" : "none"
+    enable_policy_controller = var.enable_policy_controller ? "true" : "false"
+    install_template_library = var.install_template_library ? "true" : "false"
   }
 }
 
diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl
index 8781f9db79..80155737e8 100644
--- a/modules/acm/templates/acm-config.yml.tpl
+++ b/modules/acm/templates/acm-config.yml.tpl
@@ -10,4 +10,6 @@ spec:
     syncBranch: ${sync_branch}
     secretType: ${secret_type}
     policyDir: ${policy_dir}
-
+  policyController:
+    enabled: ${enable_policy_controller}
+    templateLibraryInstalled: ${install_template_library}
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
index cc49006ae9..513556364d 100644
--- a/modules/acm/variables.tf
+++ b/modules/acm/variables.tf
@@ -61,3 +61,15 @@ variable "create_ssh_key" {
   type        = bool
   default     = true
 }
+
+variable "enable_policy_controller" {
+  description = "Whether to enable the ACM Policy Controller on the cluster"
+  type        = bool
+  default     = true
+}
+
+variable "install_template_library" {
+  description = "Whether to install the default Policy Controller template library"
+  type        = bool
+  default     = true
+}

From 3162c33ec84d31490acf87d72337d9e5bbf35764 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Fri, 15 Nov 2019 09:57:05 -0500
Subject: [PATCH 88/90] Add #268 to changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e839926e6b..e7e738732c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ Extending the adopted spec, each change should have a link to its corresponding
 * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300]
 * Support for setting node_locations on node pools. [#303]
 * Fix for specifying  `node_count` on node pools when autoscaling is disabled. [#311]
+* Added submodule for installing Anthos Config Management. [#268]
 
 ## [v5.1.1] - 2019-10-25
 
@@ -227,6 +228,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
+[#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268
 [#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311
 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303
 [#300]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/300

From 8dfd27b79700b941b9904bf97d0fe59260f112f8 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Fri, 15 Nov 2019 11:20:19 -0500
Subject: [PATCH 89/90] Update README.md

---
 modules/acm/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/acm/README.md b/modules/acm/README.md
index 166611e0b7..bef7eccd53 100644
--- a/modules/acm/README.md
+++ b/modules/acm/README.md
@@ -9,7 +9,7 @@ Specifically, this module automates the following steps for [installing ACM](htt
 
 ## Usage
 
-There is a [full example](../../examples/simple_zonal) provided. Simple usage is as follows:
+There is a [full example](../../examples/simple_zonal_with_acm) provided. Simple usage is as follows:
 
 ```tf
 module "acm" {

From efa4b61e60fe8e2bb275add05b6542dfb43cc279 Mon Sep 17 00:00:00 2001
From: Morgante Pell <morgantep@google.com>
Date: Fri, 15 Nov 2019 11:31:11 -0500
Subject: [PATCH 90/90] Don't attempt to set node_locations on pools if
 unspecified

---
 autogen/cluster.tf                                     | 2 +-
 modules/beta-private-cluster-update-variant/cluster.tf | 2 +-
 modules/beta-private-cluster/cluster.tf                | 2 +-
 modules/beta-public-cluster/cluster.tf                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/autogen/cluster.tf b/autogen/cluster.tf
index 68e23ebabc..4791220591 100644
--- a/autogen/cluster.tf
+++ b/autogen/cluster.tf
@@ -312,7 +312,7 @@ resource "google_container_node_pool" "pools" {
   location = local.location
   {% if beta_cluster %}
   // use node_locations if provided, defaults to cluster level node_locations if not specified
-  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null
   {% endif %}
   cluster  = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index a0b7f50b8d..10c206f35c 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -285,7 +285,7 @@ resource "google_container_node_pool" "pools" {
   project  = var.project_id
   location = local.location
   // use node_locations if provided, defaults to cluster level node_locations if not specified
-  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null
   cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 042a4735cd..c363dacf9f 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -213,7 +213,7 @@ resource "google_container_node_pool" "pools" {
   project  = var.project_id
   location = local.location
   // use node_locations if provided, defaults to cluster level node_locations if not specified
-  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null
   cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 2b4fb3a6e9..1f5eee84aa 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -208,7 +208,7 @@ resource "google_container_node_pool" "pools" {
   project  = var.project_id
   location = local.location
   // use node_locations if provided, defaults to cluster level node_locations if not specified
-  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations
+  node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null
   cluster        = google_container_cluster.primary.name
   version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(
     var.node_pools[count.index],