Fix issue with regional cluster roll outs causing version skews

Regional clusters are created using the newest version of GKE that is available across all zones in which the masters live. When a GKE version roll out occurs, the available versions across the zones can become skewed for zonal clusters with version x.y.z-gke.a being the only available zonal version in one zone but version x.y[+1].z[+1]-gke.a[+1] being the only zonal version available in another zone. The Terraform module only checks for the version available in the first zone returned by a call to the google_compute_zones data resource. Consequently, the module will fail to create a regional cluster during a roll out due to the version available in that zone not being available across all the zones for regional clusters.
terraform-google-modules · Mar 20, 2019 · 9c5dab5 · 9c5dab5
1 parent c105646
commit 9c5dab5
Show file tree

Hide file tree

Showing 22 changed files with 113 additions and 49 deletions.
diff --git a/README.md b/README.md
@@ -101,8 +101,8 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | - | yes |
-| ip\_range\_services | The _name_ of the secondary subnet ip range to use for services | string | - | yes |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |

diff --git a/auth.tf b/auth.tf
@@ -31,4 +31,4 @@ provider "kubernetes" {
   host                   = "https://${local.cluster_endpoint}"
   token                  = "${data.google_client_config.default.access_token}"
   cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}"
-}
+}
diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -104,7 +104,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {

diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -104,7 +104,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {

diff --git a/autogen/main.tf b/autogen/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -149,7 +151,16 @@ locals {
   Get available container engine versions
  *****************************************/
 data "google_container_engine_versions" "region" {
-  provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  provider = "google-beta"
+  region   = "${var.region}"
+  project  = "${var.project_id}"
+}
+
+data "google_container_engine_versions" "zone" {
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone     = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
   project  = "${var.project_id}"
 }
diff --git a/cluster_regional.tf b/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -97,7 +97,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {

diff --git a/cluster_zonal.tf b/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -97,7 +97,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {
@@ -107,7 +107,7 @@ resource "google_container_node_pool" "zonal_pools" {
 
   management {
     auto_repair  = "${lookup(var.node_pools[count.index], "auto_repair", true)}"
-    auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}"
+    auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}"
   }
 
   node_config {

diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   version     = "~> 1.20"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 provider "kubernetes" {
   load_config_file       = false
   host                   = "https://${module.gke.endpoint}"

diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md
@@ -17,6 +17,7 @@ This example illustrates how to create a cluster with multiple custom node-pool
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
 | subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
+| zones | The zone to host the cluster in (required if is a zonal cluster) | list | n/a | yes |
 
 ## Outputs
 

diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source                   = "../../"
   project_id               = "${var.project_id}"
@@ -43,6 +48,7 @@ module "gke" {
       min_count       = 1
       max_count       = 2
       service_account = "${var.compute_engine_service_account}"
+      auto_upgrade    = true
     },
     {
       name            = "pool-02"
@@ -53,7 +59,6 @@ module "gke" {
       disk_type       = "pd-standard"
       image_type      = "COS"
       auto_repair     = false
-      auto_upgrade    = false
       service_account = "${var.compute_engine_service_account}"
     },
   ]

diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source             = "../../"
   project_id         = "${var.project_id}"

diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf
@@ -23,6 +23,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"

diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"

diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf
@@ -24,6 +24,11 @@ provider "google" {
   region      = "${var.region}"
 }
 
+provider "google-beta" {
+  credentials = "${file(var.credentials_path)}"
+  region      = "${var.region}"
+}
+
 module "gke" {
   source            = "../../"
   project_id        = "${var.project_id}"

diff --git a/main.tf b/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -149,7 +151,17 @@ locals {
   Get available container engine versions
  *****************************************/
 data "google_container_engine_versions" "region" {
-  provider = "google"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  provider = "google-beta"
+  region   = "${var.region}"
   project  = "${var.project_id}"
-}
+}
+
+data "google_container_engine_versions" "zone" {
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
+
+  project = "${var.project_id}"
+}
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
@@ -106,8 +106,8 @@ Then perform the following commands on the root folder:
 | http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no |
 | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no |
 | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no |
-| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
-| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
+| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes |
+| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes |
 | kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no |
 | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no |
 | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no |

diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_regional}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -102,7 +102,7 @@ resource "google_container_node_pool" "pools" {
   project            = "${var.project_id}"
   region             = "${var.region}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {

diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf
@@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {
 
   network            = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
   subnetwork         = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
-  min_master_version = "${local.kubernetes_version}"
+  min_master_version = "${local.kubernetes_version_zonal}"
 
   logging_service    = "${var.logging_service}"
   monitoring_service = "${var.monitoring_service}"
@@ -102,7 +102,7 @@ resource "google_container_node_pool" "zonal_pools" {
   project            = "${var.project_id}"
   zone               = "${var.zones[0]}"
   cluster            = "${var.name}"
-  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
+  version            = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
   initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"
 
   autoscaling {

diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
@@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
 }
 
 locals {
-  kubernetes_version     = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
-  node_version           = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
-  custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
-  network_project_id     = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
+  kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
+  kubernetes_version_zonal    = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
+  node_version_regional       = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
+  node_version_zonal          = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
+  custom_kube_dns_config      = "${length(keys(var.stub_domains)) > 0 ? true : false}"
+  network_project_id          = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
 
   cluster_type = "${var.regional ? "regional" : "zonal"}"
 
@@ -150,6 +152,16 @@ locals {
  *****************************************/
 data "google_container_engine_versions" "region" {
   provider = "google-beta"
-  zone     = "${data.google_compute_zones.available.names[0]}"
+  region   = "${var.region}"
   project  = "${var.project_id}"
 }
+
+data "google_container_engine_versions" "zone" {
+  // Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
+  //
+  //     data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
+  //
+  zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
+
+  project = "${var.project_id}"
+}
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
@@ -116,7 +116,7 @@ variable "ip_range_pods" {
 }
 
 variable "ip_range_services" {
-  description = "The _name_ of the secondary subnet ip range to use for services"
+  description = "The _name_ of the secondary subnet range to use for services"
 }
 
 variable "remove_default_node_pool" {