From 1bef00742cbb2cab20525c71f1772b99ed69c3e9 Mon Sep 17 00:00:00 2001 From: kliu47 <88392374+kliu47@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:18:01 -0500 Subject: [PATCH] feat: Add additional functionality for autopilot clusters (#1985) --- autogen/main/cluster.tf.tmpl | 11 ++++----- autogen/main/main.tf.tmpl | 2 +- autogen/main/variables.tf.tmpl | 2 +- cluster.tf | 8 +++---- examples/simple_autopilot_public/main.tf | 3 +++ .../beta-autopilot-private-cluster/README.md | 3 +++ .../beta-autopilot-private-cluster/cluster.tf | 24 +++++++++++++++++++ .../beta-autopilot-private-cluster/main.tf | 3 +++ .../variables.tf | 18 ++++++++++++++ .../beta-autopilot-public-cluster/README.md | 3 +++ .../beta-autopilot-public-cluster/cluster.tf | 24 +++++++++++++++++++ modules/beta-autopilot-public-cluster/main.tf | 3 +++ .../variables.tf | 18 ++++++++++++++ .../cluster.tf | 9 ++++--- modules/beta-private-cluster/cluster.tf | 9 ++++--- .../cluster.tf | 9 ++++--- modules/beta-public-cluster/cluster.tf | 9 ++++--- .../private-cluster-update-variant/cluster.tf | 8 +++---- modules/private-cluster/cluster.tf | 8 +++---- 19 files changed, 134 insertions(+), 40 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 633ba69312..824e4942cc 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -324,6 +324,11 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + {% endif %} + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -347,13 +352,7 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } - {% endif %} {% if beta_cluster and autopilot_cluster != true %} - istio_config { disabled = !var.istio auth = var.istio_auth diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 38aa098b0b..46f408bda0 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -107,10 +107,10 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus +{% endif %} gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] -{% endif %} {% if beta_cluster and autopilot_cluster != true %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 515a8af1fd..f0c5d7d520 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -741,6 +741,7 @@ variable "gce_pd_csi_driver" { default = true } +{% endif %} variable "gke_backup_agent_config" { type = bool description = "Whether Backup for GKE agent is enabled for this cluster." @@ -759,7 +760,6 @@ variable "stateful_ha" { default = false } -{% endif %} variable "timeouts" { type = map(string) description = "Timeout for cluster operations." diff --git a/cluster.tf b/cluster.tf index 01f808aeea..2eefcc924d 100644 --- a/cluster.tf +++ b/cluster.tf @@ -244,6 +244,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -267,10 +271,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } } datapath_provider = var.datapath_provider diff --git a/examples/simple_autopilot_public/main.tf b/examples/simple_autopilot_public/main.tf index dfad96d0dc..38741490cb 100644 --- a/examples/simple_autopilot_public/main.tf +++ b/examples/simple_autopilot_public/main.tf @@ -49,4 +49,7 @@ module "gke" { network_tags = [local.cluster_type] deletion_protection = false enable_l4_ilb_subsetting = true + gcs_fuse_csi_driver = true + stateful_ha = false + gke_backup_agent_config = false } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 751d7fca52..46887b7b64 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -104,6 +104,8 @@ Then perform the following commands on the root folder: | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | | fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -141,6 +143,7 @@ Then perform the following commands on the root folder: | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index bfee2c0445..8f36118449 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -135,6 +135,30 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + + dynamic "gcs_fuse_csi_driver_config" { + for_each = local.gcs_fuse_csi_driver_config + + content { + enabled = gcs_fuse_csi_driver_config.value.enabled + } + } + + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } } allow_net_admin = var.allow_net_admin diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index f879c2d6eb..52427188dc 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -66,6 +66,9 @@ locals { cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for range in var.additional_ip_range_pods : local.cluster_alias_ranges_cidr[range] if length(range) > 0])) : [] + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] + gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 6e6f2b157a..fb7040461f 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -453,6 +453,24 @@ variable "database_encryption" { } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + +variable "gcs_fuse_csi_driver" { + type = bool + description = "Whether GCE FUSE CSI driver is enabled for this cluster." + default = false +} + +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 2adc826408..257b5b398b 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -95,6 +95,8 @@ Then perform the following commands on the root folder: | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | | fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -130,6 +132,7 @@ Then perform the following commands on the root folder: | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 8fb6118780..2945356182 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -135,6 +135,30 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + + dynamic "gcs_fuse_csi_driver_config" { + for_each = local.gcs_fuse_csi_driver_config + + content { + enabled = gcs_fuse_csi_driver_config.value.enabled + } + } + + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } } allow_net_admin = var.allow_net_admin diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index b00176d69b..3be62c142f 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -66,6 +66,9 @@ locals { cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for range in var.additional_ip_range_pods : local.cluster_alias_ranges_cidr[range] if length(range) > 0])) : [] + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] + gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 173fc99402..82a52262fc 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -423,6 +423,24 @@ variable "database_encryption" { } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + +variable "gcs_fuse_csi_driver" { + type = bool + description = "Whether GCE FUSE CSI driver is enabled for this cluster." + default = false +} + +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 359d57c007..1a54007c7d 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -267,6 +267,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -290,11 +294,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } - istio_config { disabled = !var.istio auth = var.istio_auth diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 3a3f911291..3c3e75a137 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -267,6 +267,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -290,11 +294,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } - istio_config { disabled = !var.istio auth = var.istio_auth diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 3c3f34f0cb..08c7a9fb98 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -267,6 +267,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -290,11 +294,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } - istio_config { disabled = !var.istio auth = var.istio_auth diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index fe6a6c242e..f4ce6624b8 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -267,6 +267,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -290,11 +294,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } - istio_config { disabled = !var.istio auth = var.istio_auth diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 04837e28e6..62228fbecf 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -244,6 +244,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -267,10 +271,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } } datapath_provider = var.datapath_provider diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 31573d02a9..c82ca3d399 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -244,6 +244,10 @@ resource "google_container_cluster" "primary" { } } + config_connector_config { + enabled = var.config_connector + } + dynamic "gke_backup_agent_config" { for_each = local.gke_backup_agent_config @@ -267,10 +271,6 @@ resource "google_container_cluster" "primary" { enabled = stateful_ha_config.value.enabled } } - - config_connector_config { - enabled = var.config_connector - } } datapath_provider = var.datapath_provider