diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index ffb565ff21..1ca0759a47 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -62,6 +62,18 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service {% if beta_cluster %} + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 11e0ec224f..629a47729b 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -50,6 +50,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal {% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + {% endif %} diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 5b87f08e05..3b20d3140b 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -174,8 +174,26 @@ variable "node_pools_metadata" { default-node-pool = {} } } - {% if beta_cluster %} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 237b3f0b6f..7b8867b9c8 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,6 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index b2f1d010ed..15a7c124ae 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -36,6 +36,7 @@ module "gke" { create_service_account = false remove_default_node_pool = true disable_legacy_metadata_endpoints = false + cluster_autoscaling = var.cluster_autoscaling node_pools = [ { diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 040c78d2c4..49570f241f 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -52,3 +52,20 @@ variable "compute_engine_service_account" { description = "Service account to associate to the nodes in the cluster" } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 22227f6a1e..1f4e4ea298 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 3d0c0eac34..52961dfd86 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index c0a2f6c892..02d84e73ab 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 0f581e56cc..d3f7504667 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index c5c6b079ef..528819506e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 5b75a98b2f..604fc7b2d4 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index c0a2f6c892..02d84e73ab 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 0f581e56cc..d3f7504667 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index b0e2cfc85d..e7d7820595 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -136,6 +136,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index e675d41c23..cb15fe91c6 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 84e43e33fb..6200b42633 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e27a7d0414..dc25c7a434 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 82c268aecf..1099fe5f95 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 82c268aecf..1099fe5f95 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 82dd01035c..8c787f94d6 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -26,5 +26,13 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_accounts[0] + + cluster_autoscaling = { + enabled = true + max_cpu_cores = 20 + min_cpu_cores = 5 + max_memory_gb = 30 + min_memory_gb = 10 + } } diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 69a15e8293..c08b61f1c6 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -33,6 +33,30 @@ end end + describe "cluster-autoscaling" do + it "has the expected cluster autoscaling settings" do + expect(data['autoscaling']).to eq({ + "autoprovisioningNodePoolDefaults" => { + "oauthScopes" => %w(https://www.googleapis.com/auth/logging.write https://www.googleapis.com/auth/monitoring), + "serviceAccount" => "default" + }, + "enableNodeAutoprovisioning" => true, + "resourceLimits" => [ + { + "maximum" => "20", + "minimum" => "5", + "resourceType" => "cpu" + }, + { + "maximum" => "30", + "minimum" => "10", + "resourceType" => "memory" + } + ] + }) + end + end + describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } diff --git a/variables.tf b/variables.tf index b9fdf45738..42dbc421d1 100644 --- a/variables.tf +++ b/variables.tf @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name"