diff --git a/README.md b/README.md index 9633443be2..ff954ab309 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,8 @@ Then perform the following commands on the root folder: | enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_identity\_service | Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | @@ -192,7 +194,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -201,7 +203,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -221,6 +223,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -228,8 +231,8 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -257,7 +260,9 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | | instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -293,6 +298,11 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -311,6 +321,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -325,6 +336,9 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | +| enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/autogen/main/README.md b/autogen/main/README.md index 4206111f2f..0d4c6feefc 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -197,13 +197,11 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | -{% if beta_cluster %} | cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -{% endif %} +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -225,6 +223,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -239,13 +238,11 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -{% if beta_cluster %} -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | {% if not private_cluster %} | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | {% endif %} -{% endif %} | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index a40116eab3..75ce3a1f6a 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -216,11 +216,18 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu - {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility + {% if beta_cluster %} dynamic "secret_manager_config" { for_each = var.enable_secret_manager_addon ? [var.enable_secret_manager_addon] : [] content { @@ -234,13 +241,6 @@ resource "google_container_cluster" "primary" { enabled = pod_security_policy_config.value } } - - dynamic "identity_service_config" { - for_each = var.enable_identity_service ? [var.enable_identity_service] : [] - content { - enabled = identity_service_config.value - } - } {% endif %} {% endif %} @@ -466,9 +466,12 @@ resource "google_container_cluster" "primary" { dynamic "dns_config" { for_each = var.cluster_dns_provider == "CLOUD_DNS" ? [1] : [] content { - cluster_dns = var.cluster_dns_provider - cluster_dns_scope = var.cluster_dns_scope - cluster_dns_domain = var.cluster_dns_domain + {% if beta_cluster %} + additive_vpc_scope_dns_domain = var.additive_vpc_scope_dns_domain + {% endif %} + cluster_dns = var.cluster_dns_provider + cluster_dns_scope = var.cluster_dns_scope + cluster_dns_domain = var.cluster_dns_domain } } @@ -489,9 +492,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -628,6 +632,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } {% if beta_cluster and autopilot_cluster != true %} @@ -685,6 +696,7 @@ locals { "enable_secure_boot", "boot_disk_kms_key", "queued_provisioning", + "enable_confidential_storage", ] } @@ -846,9 +858,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -993,14 +1006,6 @@ resource "google_container_node_pool" "windows_pools" { } } - {% if beta_cluster %} - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1014,6 +1019,14 @@ resource "google_container_node_pool" "windows_pools" { pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } + {% if beta_cluster %} + + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } {% endif %} {% if i == 0 %} diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 427ef73ad7..5356f7dd69 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -157,18 +157,18 @@ locals { cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service {% if autopilot_cluster != true %} - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled=false }])[0].disabled {% endif %} - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled=false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled=false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false {% if beta_cluster %} # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -225,6 +225,8 @@ locals { workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled {% if autopilot_cluster != true %} cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates @@ -238,8 +240,6 @@ locals { cluster_telemetry_type_is_set = var.cluster_telemetry_type != null {% endif %} cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 9cf774cca8..bd11d979b3 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -224,21 +224,21 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } +{% endif %} output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } -{% endif %} output "fleet_membership" { description = "Fleet membership (if registered)" diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index dea04e2728..86e576c9f5 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -576,13 +576,13 @@ variable "enable_confidential_nodes" { {% if beta_cluster %} variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -602,13 +602,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -620,9 +620,15 @@ variable "disable_default_snat" { } variable "notification_config_topic" { - type = string + type = string description = "The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}." - default = "" + default = "" +} + +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] } variable "deletion_protection" { @@ -735,6 +741,13 @@ variable "cluster_dns_domain" { default = "" } +{% if beta_cluster %} +variable "additive_vpc_scope_dns_domain" { + type = string + description = "(Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster_dns = `CLOUD_DNS` and cluster_dns_scope = `CLUSTER_SCOPE` must both be set as well." + default = "" +} +{% endif %} variable "gce_pd_csi_driver" { type = bool description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." @@ -795,13 +808,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -817,6 +830,12 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + {% endif %} variable "enable_l4_ilb_subsetting" { type = bool @@ -874,24 +893,20 @@ variable "sandbox_enabled" { default = false } -variable "enable_intranode_visibility" { +variable "enable_gcfs" { type = bool - description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + description = "Enable image streaming on cluster level." default = false } + {% endif %} +{% endif %} +{% if autopilot_cluster != true %} variable "enable_identity_service" { type = bool description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "enable_gcfs" { - type = bool - description = "Enable image streaming on cluster level." - default = false -} - {% endif %} {% endif %} {% if autopilot_cluster %} variable "allow_net_admin" { diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 13306c87b8..47ece2199e 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -535,6 +535,26 @@ steps: - verify simple-autopilot-private-non-default-sa name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose'] +- id: init simple-fleet-app-operator-permissions + waitFor: + - create-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage init --verbose'] +- id: apply simple-fleet-app-operator-permissions + waitFor: + - init simple-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage apply --verbose'] +- id: verify simple-fleet-app-operator-permissions + waitFor: + - apply simple-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage verify --verbose'] +- id: teardown simple-fleet-app-operator-permissions + waitFor: + - verify simple-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/cluster.tf b/cluster.tf index 2eefcc924d..3ea59eddca 100644 --- a/cluster.tf +++ b/cluster.tf @@ -173,8 +173,17 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - enable_tpu = var.enable_tpu + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting @@ -362,9 +371,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -463,6 +473,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } } @@ -554,9 +571,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -693,6 +711,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -813,9 +844,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -952,6 +984,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/examples/simple_fleet_app_operator_permissions/README.md b/examples/simple_fleet_app_operator_permissions/README.md new file mode 100644 index 0000000000..6de1c4e59d --- /dev/null +++ b/examples/simple_fleet_app_operator_permissions/README.md @@ -0,0 +1,26 @@ +# Simple App Operator Permissions Setup for a Fleet Scope + +This example illustrates how to create a Fleet Scope for a [team](https://cloud.google.com/kubernetes-engine/fleet-management/docs/team-management) and set up permissions for an app operator in the team. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| fleet\_project\_id | The project to which the Fleet belongs. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| fleet\_project\_id | The project to which the Fleet belongs. | +| wait | An output (Fleet Scope RBAC Role Binding IDs) to use when you want to depend on granting permissions finishing. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf new file mode 100644 index 0000000000..e905f8bc2e --- /dev/null +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + app_operator_id = "app-operator-id" + app_operator_team = "app-operator-team" + app_operator_role = "VIEW" +} + +# Create a Service Account, which can be used as an app operator. +resource "google_service_account" "service_account" { + project = var.fleet_project_id + account_id = local.app_operator_id + display_name = "Test App Operator Service Account" +} + +# Create a Fleet Scope for the app operator's team. +resource "google_gke_hub_scope" "scope" { + project = var.fleet_project_id + scope_id = local.app_operator_team +} + +# Grant permissions to the app operator to work with the Fleet Scope. +module "permissions" { + source = "../../modules/fleet-app-operator-permissions" + + fleet_project_id = var.fleet_project_id + scope_id = google_gke_hub_scope.scope.scope_id + users = ["${local.app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"] + groups = [] + role = local.app_operator_role + + depends_on = [ + google_service_account.service_account + ] +} + diff --git a/examples/simple_fleet_app_operator_permissions/outputs.tf b/examples/simple_fleet_app_operator_permissions/outputs.tf new file mode 100644 index 0000000000..d0e55d96eb --- /dev/null +++ b/examples/simple_fleet_app_operator_permissions/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "fleet_project_id" { + description = "The project to which the Fleet belongs." + value = var.fleet_project_id +} + +output "wait" { + description = "An output (Fleet Scope RBAC Role Binding IDs) to use when you want to depend on granting permissions finishing." + value = module.permissions.wait +} + diff --git a/examples/simple_fleet_app_operator_permissions/variables.tf b/examples/simple_fleet_app_operator_permissions/variables.tf new file mode 100644 index 0000000000..93185b8300 --- /dev/null +++ b/examples/simple_fleet_app_operator_permissions/variables.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "fleet_project_id" { + description = "The project to which the Fleet belongs." + type = string +} + diff --git a/examples/simple_fleet_app_operator_permissions/versions.tf b/examples/simple_fleet_app_operator_permissions/versions.tf new file mode 100644 index 0000000000..fe82a4e387 --- /dev/null +++ b/examples/simple_fleet_app_operator_permissions/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.2.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.81.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.81.0" + } + } +} + diff --git a/main.tf b/main.tf index f41e968be5..f7bb8b486b 100644 --- a/main.tf +++ b/main.tf @@ -118,10 +118,12 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ @@ -165,7 +167,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 71a415fa3c..7ee5aebd0c 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -130,14 +130,15 @@ Then perform the following commands on the root folder: | network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | | registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -149,8 +150,8 @@ Then perform the following commands on the root folder: | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 331ce7b1a9..a7283e1e35 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -296,6 +296,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index 19355a74dc..836c151bc1 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -87,15 +87,15 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -128,13 +128,13 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index 9d3954fb45..66610c3682 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -179,9 +179,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -189,9 +189,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 017f13766e..93212a7de1 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -382,13 +382,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -407,13 +407,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -430,6 +430,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -486,6 +492,7 @@ variable "enable_l4_ilb_subsetting" { description = "Enable L4 ILB Subsetting on the cluster" default = false } + variable "allow_net_admin" { description = "(Optional) Enable NET_ADMIN for the cluster." type = bool diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index d29fa7a7ae..17d495b60c 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -119,14 +119,15 @@ Then perform the following commands on the root folder: | network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | | registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -138,8 +139,8 @@ Then perform the following commands on the root folder: | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 57b9288fcf..ff4c88b6f2 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -277,6 +277,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 9c89bd9b67..fb8af5bef8 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -86,15 +86,15 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -127,13 +127,13 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index f23b1ed6ac..98cf3fbdd3 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -169,9 +169,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -179,9 +179,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index f73b556115..f3cf4f094e 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -352,13 +352,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -377,13 +377,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -400,6 +400,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -456,6 +462,7 @@ variable "enable_l4_ilb_subsetting" { description = "Enable L4 ILB Subsetting on the cluster" default = false } + variable "allow_net_admin" { description = "(Optional) Enable NET_ADMIN for the cluster." type = bool diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 7f0575c31d..9c39988304 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -173,6 +173,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | (Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | @@ -239,7 +240,7 @@ Then perform the following commands on the root folder: | istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | | kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -250,7 +251,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -270,6 +271,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -278,8 +280,8 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -292,8 +294,8 @@ Then perform the following commands on the root folder: | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs @@ -357,7 +359,7 @@ The node_pools variable takes the following parameters: | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -377,6 +379,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -391,7 +394,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 630ccc6669..cd3158132c 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -180,6 +180,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu enable_intranode_visibility = var.enable_intranode_visibility @@ -198,13 +205,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "identity_service_config" { - for_each = var.enable_identity_service ? [var.enable_identity_service] : [] - content { - enabled = identity_service_config.value - } - } - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting enable_cilium_clusterwide_network_policy = var.enable_cilium_clusterwide_network_policy @@ -395,9 +395,10 @@ resource "google_container_cluster" "primary" { dynamic "dns_config" { for_each = var.cluster_dns_provider == "CLOUD_DNS" ? [1] : [] content { - cluster_dns = var.cluster_dns_provider - cluster_dns_scope = var.cluster_dns_scope - cluster_dns_domain = var.cluster_dns_domain + additive_vpc_scope_dns_domain = var.additive_vpc_scope_dns_domain + cluster_dns = var.cluster_dns_provider + cluster_dns_scope = var.cluster_dns_scope + cluster_dns_domain = var.cluster_dns_domain } } @@ -416,9 +417,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -544,6 +546,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } @@ -586,6 +595,7 @@ locals { "enable_secure_boot", "boot_disk_kms_key", "queued_provisioning", + "enable_confidential_storage", ] } @@ -724,9 +734,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -869,13 +880,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -890,6 +894,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -1010,9 +1021,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -1155,13 +1167,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1176,6 +1181,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 96267061f4..1bfa3658a5 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -131,16 +131,16 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -186,7 +186,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -195,8 +197,6 @@ locals { cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 3702c4b729..99a88a3264 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -205,9 +205,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -215,9 +215,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index aa76cf0a3f..cf55ef1360 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -551,13 +551,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -576,13 +576,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -599,6 +599,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -704,6 +710,11 @@ variable "cluster_dns_domain" { default = "" } +variable "additive_vpc_scope_dns_domain" { + type = string + description = "(Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster_dns = `CLOUD_DNS` and cluster_dns_scope = `CLUSTER_SCOPE` must both be set as well." + default = "" +} variable "gce_pd_csi_driver" { type = bool description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." @@ -762,13 +773,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -784,6 +795,12 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" @@ -838,9 +855,9 @@ variable "sandbox_enabled" { default = false } -variable "enable_intranode_visibility" { +variable "enable_gcfs" { type = bool - description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + description = "Enable image streaming on cluster level." default = false } @@ -850,12 +867,6 @@ variable "enable_identity_service" { default = false } -variable "enable_gcfs" { - type = bool - description = "Enable image streaming on cluster level." - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index e6faa6b955..3e239a08ea 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -151,6 +151,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | (Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | @@ -217,7 +218,7 @@ Then perform the following commands on the root folder: | istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | | kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -228,7 +229,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -248,6 +249,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -256,8 +258,8 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -270,8 +272,8 @@ Then perform the following commands on the root folder: | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs @@ -335,7 +337,7 @@ The node_pools variable takes the following parameters: | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -355,6 +357,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -369,7 +372,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 88469dd152..5154693a3a 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -180,6 +180,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu enable_intranode_visibility = var.enable_intranode_visibility @@ -198,13 +205,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "identity_service_config" { - for_each = var.enable_identity_service ? [var.enable_identity_service] : [] - content { - enabled = identity_service_config.value - } - } - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting enable_cilium_clusterwide_network_policy = var.enable_cilium_clusterwide_network_policy @@ -395,9 +395,10 @@ resource "google_container_cluster" "primary" { dynamic "dns_config" { for_each = var.cluster_dns_provider == "CLOUD_DNS" ? [1] : [] content { - cluster_dns = var.cluster_dns_provider - cluster_dns_scope = var.cluster_dns_scope - cluster_dns_domain = var.cluster_dns_domain + additive_vpc_scope_dns_domain = var.additive_vpc_scope_dns_domain + cluster_dns = var.cluster_dns_provider + cluster_dns_scope = var.cluster_dns_scope + cluster_dns_domain = var.cluster_dns_domain } } @@ -416,9 +417,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -544,6 +546,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } @@ -645,9 +654,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -790,13 +800,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -811,6 +814,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -930,9 +940,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -1075,13 +1086,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1096,6 +1100,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 96267061f4..1bfa3658a5 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -131,16 +131,16 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -186,7 +186,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -195,8 +197,6 @@ locals { cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 3702c4b729..99a88a3264 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -205,9 +205,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -215,9 +215,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index aa76cf0a3f..cf55ef1360 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -551,13 +551,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -576,13 +576,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -599,6 +599,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -704,6 +710,11 @@ variable "cluster_dns_domain" { default = "" } +variable "additive_vpc_scope_dns_domain" { + type = string + description = "(Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster_dns = `CLOUD_DNS` and cluster_dns_scope = `CLUSTER_SCOPE` must both be set as well." + default = "" +} variable "gce_pd_csi_driver" { type = bool description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." @@ -762,13 +773,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -784,6 +795,12 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" @@ -838,9 +855,9 @@ variable "sandbox_enabled" { default = false } -variable "enable_intranode_visibility" { +variable "enable_gcfs" { type = bool - description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + description = "Enable image streaming on cluster level." default = false } @@ -850,12 +867,6 @@ variable "enable_identity_service" { default = false } -variable "enable_gcfs" { - type = bool - description = "Enable image streaming on cluster level." - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 780d094616..3c728f9282 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -167,6 +167,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | (Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | @@ -230,7 +231,7 @@ Then perform the following commands on the root folder: | istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | | kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -239,7 +240,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -259,6 +260,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -267,8 +269,8 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -281,8 +283,8 @@ Then perform the following commands on the root folder: | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs @@ -344,7 +346,7 @@ The node_pools variable takes the following parameters: | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -364,6 +366,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -378,7 +381,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 91608910c7..93339ad6e2 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -180,6 +180,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu enable_intranode_visibility = var.enable_intranode_visibility @@ -198,13 +205,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "identity_service_config" { - for_each = var.enable_identity_service ? [var.enable_identity_service] : [] - content { - enabled = identity_service_config.value - } - } - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting enable_cilium_clusterwide_network_policy = var.enable_cilium_clusterwide_network_policy @@ -395,9 +395,10 @@ resource "google_container_cluster" "primary" { dynamic "dns_config" { for_each = var.cluster_dns_provider == "CLOUD_DNS" ? [1] : [] content { - cluster_dns = var.cluster_dns_provider - cluster_dns_scope = var.cluster_dns_scope - cluster_dns_domain = var.cluster_dns_domain + additive_vpc_scope_dns_domain = var.additive_vpc_scope_dns_domain + cluster_dns = var.cluster_dns_provider + cluster_dns_scope = var.cluster_dns_scope + cluster_dns_domain = var.cluster_dns_domain } } @@ -416,9 +417,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -525,6 +527,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } @@ -567,6 +576,7 @@ locals { "enable_secure_boot", "boot_disk_kms_key", "queued_provisioning", + "enable_confidential_storage", ] } @@ -705,9 +715,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -850,13 +861,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -871,6 +875,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -991,9 +1002,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -1136,13 +1148,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1157,6 +1162,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index f435b715cb..0039ec882f 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -130,16 +130,16 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -185,7 +185,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -194,8 +196,6 @@ locals { cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index c45e89b6df..9747767770 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -195,9 +195,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -205,9 +205,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 5601d6af18..b14af0abbd 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -521,13 +521,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -546,13 +546,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -569,6 +569,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -674,6 +680,11 @@ variable "cluster_dns_domain" { default = "" } +variable "additive_vpc_scope_dns_domain" { + type = string + description = "(Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster_dns = `CLOUD_DNS` and cluster_dns_scope = `CLUSTER_SCOPE` must both be set as well." + default = "" +} variable "gce_pd_csi_driver" { type = bool description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." @@ -732,13 +743,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -754,6 +765,12 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" @@ -808,9 +825,9 @@ variable "sandbox_enabled" { default = false } -variable "enable_intranode_visibility" { +variable "enable_gcfs" { type = bool - description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + description = "Enable image streaming on cluster level." default = false } @@ -820,12 +837,6 @@ variable "enable_identity_service" { default = false } -variable "enable_gcfs" { - type = bool - description = "Enable image streaming on cluster level." - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 4fc82a77dc..92f328b9bb 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -145,6 +145,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | (Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | @@ -208,7 +209,7 @@ Then perform the following commands on the root folder: | istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | | kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -217,7 +218,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -237,6 +238,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -245,8 +247,8 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -259,8 +261,8 @@ Then perform the following commands on the root folder: | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | ## Outputs @@ -322,7 +324,7 @@ The node_pools variable takes the following parameters: | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -342,6 +344,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -356,7 +359,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 102b17e3da..bd8672dde9 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -180,6 +180,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu enable_intranode_visibility = var.enable_intranode_visibility @@ -198,13 +205,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "identity_service_config" { - for_each = var.enable_identity_service ? [var.enable_identity_service] : [] - content { - enabled = identity_service_config.value - } - } - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting enable_cilium_clusterwide_network_policy = var.enable_cilium_clusterwide_network_policy @@ -395,9 +395,10 @@ resource "google_container_cluster" "primary" { dynamic "dns_config" { for_each = var.cluster_dns_provider == "CLOUD_DNS" ? [1] : [] content { - cluster_dns = var.cluster_dns_provider - cluster_dns_scope = var.cluster_dns_scope - cluster_dns_domain = var.cluster_dns_domain + additive_vpc_scope_dns_domain = var.additive_vpc_scope_dns_domain + cluster_dns = var.cluster_dns_provider + cluster_dns_scope = var.cluster_dns_scope + cluster_dns_domain = var.cluster_dns_domain } } @@ -416,9 +417,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -525,6 +527,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } @@ -626,9 +635,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -771,13 +781,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -792,6 +795,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -911,9 +921,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -1056,13 +1067,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1077,6 +1081,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index f435b715cb..0039ec882f 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -130,16 +130,16 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config[0].istio_config != null && length(google_container_cluster.primary.addons_config[0].istio_config) == 1 ? google_container_cluster.primary.addons_config[0].istio_config[0].disabled : false cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config[0].enabled : false - cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false cluster_output_secret_manager_addon_enabled = google_container_cluster.primary.secret_manager_config != null && length(google_container_cluster.primary.secret_manager_config) == 1 ? google_container_cluster.primary.secret_manager_config[0].enabled : false # /BETA features @@ -185,7 +185,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -194,8 +196,6 @@ locals { cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled - cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_secret_manager_addon_enabled = local.cluster_output_secret_manager_addon_enabled # /BETA features diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index c45e89b6df..9747767770 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -195,9 +195,9 @@ output "pod_security_policy_enabled" { value = local.cluster_pod_security_policy_enabled } -output "intranode_visibility_enabled" { - description = "Whether intra-node visibility is enabled" - value = local.cluster_intranode_visibility_enabled +output "secret_manager_addon_enabled" { + description = "Whether Secret Manager add-on is enabled" + value = local.cluster_secret_manager_addon_enabled } output "identity_service_enabled" { @@ -205,9 +205,9 @@ output "identity_service_enabled" { value = local.cluster_identity_service_enabled } -output "secret_manager_addon_enabled" { - description = "Whether Secret Manager add-on is enabled" - value = local.cluster_secret_manager_addon_enabled +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled } output "fleet_membership" { diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 5601d6af18..b14af0abbd 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -521,13 +521,13 @@ variable "enable_confidential_nodes" { } variable "workload_vulnerability_mode" { - description = "(beta) Vulnerability mode." + description = "(beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC." type = string default = "" } variable "workload_config_audit_mode" { - description = "(beta) Workload config audit mode." + description = "(beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC." type = string default = "DISABLED" } @@ -546,13 +546,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -569,6 +569,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -674,6 +680,11 @@ variable "cluster_dns_domain" { default = "" } +variable "additive_vpc_scope_dns_domain" { + type = string + description = "(Beta) This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster_dns = `CLOUD_DNS` and cluster_dns_scope = `CLUSTER_SCOPE` must both be set as well." + default = "" +} variable "gce_pd_csi_driver" { type = bool description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." @@ -732,13 +743,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -754,6 +765,12 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" @@ -808,9 +825,9 @@ variable "sandbox_enabled" { default = false } -variable "enable_intranode_visibility" { +variable "enable_gcfs" { type = bool - description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + description = "Enable image streaming on cluster level." default = false } @@ -820,12 +837,6 @@ variable "enable_identity_service" { default = false } -variable "enable_gcfs" { - type = bool - description = "Enable image streaming on cluster level." - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/fleet-app-operator-permissions/README.md b/modules/fleet-app-operator-permissions/README.md new file mode 100644 index 0000000000..085fcb4c9c --- /dev/null +++ b/modules/fleet-app-operator-permissions/README.md @@ -0,0 +1,44 @@ +# Terrafrom Module for Fleet App Operator Permissions + +This module bundles different permissions (IAM and RBAC Role Bindings) required for [Fleet team management](https://cloud.google.com/kubernetes-engine/fleet-management/docs/team-management). A platform admin can use this module to set up permissions for an app operator (user or group) in a team--including usage of Fleet Scopes, Connect Gateway, logging, and metrics--based on predefined roles (VIEW, EDIT, ADMIN). + +## Usage +```tf +Example: +module "fleet_app_operator_permissions" { + source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions" + + fleet_project_id = "my-project-id" + scope_id = "frontend-team" + users = ["person1@company.com", "person2@company.com"] + groups = ["people@company.com"] + role = "EDIT" +} +``` + +To deploy this config, run: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| fleet\_project\_id | The project to which the Fleet belongs. | `string` | n/a | yes | +| groups | The list of app operator group principals, e.g., `people@google.com`, `principalSet://iam.googleapis.com/locations/global/workforcePools/my-pool/group/people`. | `list(string)` | n/a | yes | +| role | The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). | `string` | n/a | yes | +| scope\_id | The scope for which IAM and RBAC role bindings are created. | `string` | n/a | yes | +| users | The list of app operator user principals, e.g., `person@google.com`, `principal://iam.googleapis.com/locations/global/workforcePools/my-pool/subject/person`, `serviceAccount:my-service-account@my-project.iam.gserviceaccount.com`. | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| fleet\_project\_id | The project to which the Fleet belongs. | +| wait | An output to use when you want to depend on Scope RBAC Role Binding creation finishing. | + + diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf new file mode 100644 index 0000000000..81fc13a96c --- /dev/null +++ b/modules/fleet-app-operator-permissions/main.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + user_principals = [for name in var.users : ( + startswith(name, "principal://") ? name : ( + endswith(name, "gserviceaccount.com") ? "serviceAccount:${name}" : ( + "user:${name}" + )))] + + group_principals = [for name in var.groups : ( + startswith(name, "principalSet://") ? name : ( + "group:${name}" + ))] + + project_level_scope_role = { + "VIEW" = "roles/gkehub.scopeViewerProjectLevel" + "EDIT" = "roles/gkehub.scopeEditorProjectLevel" + "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT + } + + resource_level_scope_role = { + "VIEW" = "roles/gkehub.scopeViewer" + "EDIT" = "roles/gkehub.scopeEditor" + "ADMIN" = "roles/gkehub.scopeAdmin" + } +} + +resource "google_project_iam_binding" "log_view_permissions" { + project = var.fleet_project_id + role = "roles/logging.viewAccessor" + members = concat(local.user_principals, local.group_principals) + condition { + title = "conditional log view access" + description = "log view access for scope ${var.scope_id}" + expression = "resource.name == \"projects/${var.fleet_project_id}/locations/global/buckets/fleet-o11y-scope-${var.scope_id}/views/fleet-o11y-scope-${var.scope_id}-k8s_container\" || resource.name == \"projects/${var.fleet_project_id}/locations/global/buckets/fleet-o11y-scope-${var.scope_id}/views/fleet-o11y-scope-${var.scope_id}-k8s_pod\"" + } +} + +resource "google_project_iam_binding" "project_level_scope_permissions" { + project = var.fleet_project_id + role = local.project_level_scope_role[var.role] + members = concat(local.user_principals, local.group_principals) +} + +resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { + project = var.fleet_project_id + scope_id = var.scope_id + role = local.resource_level_scope_role[var.role] + members = concat(local.user_principals, local.group_principals) +} + +resource "random_id" "user_rand_suffix" { + for_each = toset(var.users) + byte_length = 4 +} + +resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings" { + for_each = toset(var.users) + project = var.fleet_project_id + scope_rbac_role_binding_id = "tf-${substr(join("", regexall("[a-z0-9]+", each.key)), 0, 16)}-${random_id.user_rand_suffix[each.key].hex}" + scope_id = var.scope_id + user = each.key + role { + predefined_role = var.role + } +} + +resource "random_id" "group_rand_suffix" { + for_each = toset(var.groups) + byte_length = 4 +} + +resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_bindings" { + for_each = toset(var.groups) + project = var.fleet_project_id + scope_rbac_role_binding_id = "tf-${substr(join("", regexall("[a-z0-9]+", each.key)), 0, 16)}-${random_id.group_rand_suffix[each.key].hex}" + scope_id = var.scope_id + group = each.key + role { + predefined_role = var.role + } +} + diff --git a/modules/fleet-app-operator-permissions/outputs.tf b/modules/fleet-app-operator-permissions/outputs.tf new file mode 100644 index 0000000000..8b4fbfa5fd --- /dev/null +++ b/modules/fleet-app-operator-permissions/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "fleet_project_id" { + description = "The project to which the Fleet belongs." + value = var.fleet_project_id +} + +output "wait" { + description = "An output to use when you want to depend on Scope RBAC Role Binding creation finishing." + value = { + for k, v in merge(google_gke_hub_scope_rbac_role_binding.scope_rbac_user_role_bindings, google_gke_hub_scope_rbac_role_binding.scope_rbac_group_role_bindings) : k => v.scope_rbac_role_binding_id + } +} + diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf new file mode 100644 index 0000000000..88f27d376f --- /dev/null +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "fleet_project_id" { + description = "The project to which the Fleet belongs." + type = string +} + +variable "scope_id" { + description = "The scope for which IAM and RBAC role bindings are created." + type = string +} + +variable "users" { + description = "The list of app operator user principals, e.g., `person@google.com`, `principal://iam.googleapis.com/locations/global/workforcePools/my-pool/subject/person`, `serviceAccount:my-service-account@my-project.iam.gserviceaccount.com`." + type = list(string) +} + +variable "groups" { + description = "The list of app operator group principals, e.g., `people@google.com`, `principalSet://iam.googleapis.com/locations/global/workforcePools/my-pool/group/people`." + type = list(string) +} + +variable "role" { + description = "The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)." + type = string + validation { + condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) + error_message = "Allowed values for role are VIEW, EDIT, or ADMIN." + } +} + diff --git a/modules/fleet-app-operator-permissions/versions.tf b/modules/fleet-app-operator-permissions/versions.tf new file mode 100644 index 0000000000..8903345713 --- /dev/null +++ b/modules/fleet-app-operator-permissions/versions.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.2.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.81.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.81.0" + } + random = { + source = "hashicorp/random" + version = ">= 2.0.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:fleet-app-operator-permissions/v30.1.0" + } +} + diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index b3ee327805..711905001c 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -194,6 +194,8 @@ Then perform the following commands on the root folder: | enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_identity\_service | Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | @@ -223,7 +225,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -234,7 +236,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -254,6 +256,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -261,8 +264,8 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -290,7 +293,9 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | | instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -328,6 +333,11 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -346,6 +356,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -360,6 +371,8 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 62228fbecf..e028ca6e19 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -173,8 +173,17 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - enable_tpu = var.enable_tpu + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting @@ -362,9 +371,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -482,6 +492,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } } @@ -514,6 +531,7 @@ locals { "enable_secure_boot", "boot_disk_kms_key", "queued_provisioning", + "enable_confidential_storage", ] } @@ -652,9 +670,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -791,6 +810,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -912,9 +944,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -1051,6 +1084,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 7c7de2529f..90702177dd 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -119,10 +119,12 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ @@ -166,7 +168,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 8f35e71fe0..eeec118921 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -190,6 +190,16 @@ output "dns_cache_enabled" { value = local.cluster_dns_cache_enabled } +output "identity_service_enabled" { + description = "Whether Identity Service is enabled" + value = local.cluster_identity_service_enabled +} + +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index d032c417db..5ab8307468 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -551,13 +551,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -574,6 +574,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -737,13 +743,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -759,12 +765,24 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" default = false } +variable "enable_identity_service" { + type = bool + description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 4934cf9c64..693bae3bc0 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -172,6 +172,8 @@ Then perform the following commands on the root folder: | enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_identity\_service | Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | @@ -201,7 +203,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -212,7 +214,7 @@ Then perform the following commands on the root folder: | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -232,6 +234,7 @@ Then perform the following commands on the root folder: | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | @@ -239,8 +242,8 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | @@ -268,7 +271,9 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | | instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -306,6 +311,11 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | +| enable_confidential_nodes | An optional flag to enable confidential node config. | false | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -324,6 +334,7 @@ The node_pools variable takes the following parameters: | local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | +| enable_confidential_storage | Enabling Confidential Storage will create boot disk with confidential mode. | false | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | @@ -338,6 +349,8 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index c82ca3d399..234ece4b0e 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -173,8 +173,17 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - enable_tpu = var.enable_tpu + dynamic "identity_service_config" { + for_each = var.enable_identity_service ? [var.enable_identity_service] : [] + content { + enabled = identity_service_config.value + } + } + + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting @@ -362,9 +371,10 @@ resource "google_container_cluster" "primary" { } node_config { - image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") - machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + image_type = lookup(var.node_pools[0], "image_type", "COS_CONTAINERD") + machine_type = lookup(var.node_pools[0], "machine_type", "e2-medium") + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(var.node_pools[0], "enable_gcfs", false) ? [true] : [] content { @@ -482,6 +492,13 @@ resource "google_container_cluster" "primary" { pubsub { enabled = var.notification_config_topic != "" ? true : false topic = var.notification_config_topic + + dynamic "filter" { + for_each = length(var.notification_filter_event_type) > 0 ? [1] : [] + content { + event_type = var.notification_filter_event_type + } + } } } } @@ -573,9 +590,10 @@ resource "google_container_node_pool" "pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -712,6 +730,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -832,9 +863,10 @@ resource "google_container_node_pool" "windows_pools" { } node_config { - image_type = lookup(each.value, "image_type", "COS_CONTAINERD") - machine_type = lookup(each.value, "machine_type", "e2-medium") - min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + image_type = lookup(each.value, "image_type", "COS_CONTAINERD") + machine_type = lookup(each.value, "machine_type", "e2-medium") + min_cpu_platform = lookup(each.value, "min_cpu_platform", "") + enable_confidential_storage = lookup(var.node_pools[0], "enable_confidential_storage", false) dynamic "gcfs_config" { for_each = lookup(each.value, "enable_gcfs", false) ? [true] : [] content { @@ -971,6 +1003,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 7c7de2529f..90702177dd 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -119,10 +119,12 @@ locals { cluster_output_min_master_version = google_container_cluster.primary.min_master_version cluster_output_logging_service = google_container_cluster.primary.logging_service cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service - cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config[0].network_policy_config[0].disabled - cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config[0].http_load_balancing[0].disabled - cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config[0].horizontal_pod_autoscaling[0].disabled + cluster_output_network_policy_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "network_policy_config", [{}]), [{ disabled = false }])[0].disabled + cluster_output_http_load_balancing_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "http_load_balancing", [{}]), [{ disabled = false }])[0].disabled + cluster_output_horizontal_pod_autoscaling_enabled = coalescelist(lookup(coalescelist(google_container_cluster.primary.addons_config, [{}])[0], "horizontal_pod_autoscaling", [{}]), [{ disabled = false }])[0].disabled cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling[0].enabled : false + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_identity_service_enabled = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config[0].enabled : false master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ @@ -166,7 +168,9 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_identity_service_enabled = local.cluster_output_identity_service_enabled cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 8f35e71fe0..eeec118921 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -190,6 +190,16 @@ output "dns_cache_enabled" { value = local.cluster_dns_cache_enabled } +output "identity_service_enabled" { + description = "Whether Identity Service is enabled" + value = local.cluster_identity_service_enabled +} + +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index d032c417db..5ab8307468 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -551,13 +551,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -574,6 +574,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -737,13 +743,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -759,12 +765,24 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" default = false } +variable "enable_identity_service" { + type = bool + description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/outputs.tf b/outputs.tf index 7aa1c3be10..c6a853f743 100644 --- a/outputs.tf +++ b/outputs.tf @@ -180,6 +180,16 @@ output "dns_cache_enabled" { value = local.cluster_dns_cache_enabled } +output "identity_service_enabled" { + description = "Whether Identity Service is enabled" + value = local.cluster_identity_service_enabled +} + +output "intranode_visibility_enabled" { + description = "Whether intra-node visibility is enabled" + value = local.cluster_intranode_visibility_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/test/fixtures/simple_fleet_app_operator_permissions/main.tf b/test/fixtures/simple_fleet_app_operator_permissions/main.tf new file mode 100644 index 0000000000..e0f036feea --- /dev/null +++ b/test/fixtures/simple_fleet_app_operator_permissions/main.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + project_id = var.project_ids[3] # gke-project-fleet +} + +module "example" { + source = "../../../examples/simple_fleet_app_operator_permissions" + + fleet_project_id = local.project_id +} + diff --git a/test/fixtures/simple_fleet_app_operator_permissions/outputs.tf b/test/fixtures/simple_fleet_app_operator_permissions/outputs.tf new file mode 100644 index 0000000000..eb65499ec3 --- /dev/null +++ b/test/fixtures/simple_fleet_app_operator_permissions/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = local.project_id +} + diff --git a/test/fixtures/simple_fleet_app_operator_permissions/variables.tf b/test/fixtures/simple_fleet_app_operator_permissions/variables.tf new file mode 100644 index 0000000000..27b7df966e --- /dev/null +++ b/test/fixtures/simple_fleet_app_operator_permissions/variables.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + diff --git a/test/integration/go.mod b/test/integration/go.mod index 4224097241..1aea1894ab 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.5 require ( github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.16.0 - github.com/gruntwork-io/terratest v0.46.16 + github.com/gruntwork-io/terratest v0.47.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 ) diff --git a/test/integration/go.sum b/test/integration/go.sum index 66a92cb291..de2010ae10 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -378,8 +378,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= -github.com/gruntwork-io/terratest v0.46.16 h1:l+HHuU7lNLwoAl2sP8zkYJy0uoE2Mwha2nw+rim+OhQ= -github.com/gruntwork-io/terratest v0.46.16/go.mod h1:oywHw1cFKXSYvKPm27U7quZVzDUlA22H2xUrKCe26xM= +github.com/gruntwork-io/terratest v0.47.0 h1:xIy1pT7NbGVlMLDZEHl3+3iSnvffh8tN2pL6idn448c= +github.com/gruntwork-io/terratest v0.47.0/go.mod h1:oywHw1cFKXSYvKPm27U7quZVzDUlA22H2xUrKCe26xM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go new file mode 100644 index 0000000000..23d9b46ec7 --- /dev/null +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_fleet_app_operator_permissions + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" + ) + +func TestSimpleFleetAppOperatorPermissions(t *testing.T) { + appOppT := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + appOppT.DefineVerify(func(assert *assert.Assertions) { + projectId := appOppT.GetStringOutput("project_id") + scopeId := "app-operator-team" + appOperatorEmail := fmt.Sprintf("app-operator-id@%s.iam.gserviceaccount.com", projectId) + appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) + scopeLevelRole := "roles/gkehub.scopeViewer" + projectLevelRole := "roles/gkehub.scopeViewerProjectLevel" + logViewRole := "roles/logging.viewAccessor" + logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId) + logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId) + + scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String() + assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings") + + scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String() + assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy") + assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy") + + projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String() + assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy") + }) + + appOppT.Test() +} + diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 45c4268327..fe97685dd8 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -104,6 +104,14 @@ resource "google_project_iam_member" "int_test_asm" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "int_test_fleet" { + for_each = toset(local.int_required_roles) + + project = module.gke-project-fleet.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_service_account_key" "int_test" { service_account_id = google_service_account.int_test.id } diff --git a/test/setup/main.tf b/test/setup/main.tf index f0b3e278a6..fb10513a64 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -103,3 +103,19 @@ module "gke-project-asm" { activate_apis = local.apis } + +module "gke-project-fleet" { + source = "terraform-google-modules/project-factory/google" + version = "~> 15.0" + + name = "ci-gke-fleet-${random_id.random_project_id_suffix.hex}" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + # due to https://github.com/hashicorp/terraform-provider-google/issues/9505 for AP + default_service_account = "keep" + + activate_apis = local.apis +} + diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index b14d5daac6..54c01df2e7 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -15,7 +15,7 @@ */ output "project_ids" { - value = [module.gke-project-1.project_id, module.gke-project-2.project_id, module.gke-project-asm.project_id] + value = [module.gke-project-1.project_id, module.gke-project-2.project_id, module.gke-project-asm.project_id, module.gke-project-fleet.project_id] } output "sa_key" { diff --git a/variables.tf b/variables.tf index d36fe30142..8c37d4e4fd 100644 --- a/variables.tf +++ b/variables.tf @@ -521,13 +521,13 @@ variable "enable_cilium_clusterwide_network_policy" { } variable "security_posture_mode" { - description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." + description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." type = string default = "DISABLED" } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -544,6 +544,12 @@ variable "notification_config_topic" { default = "" } +variable "notification_filter_event_type" { + type = list(string) + description = "Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE_AVAILABLE_EVENT, UPGRADE_EVENT, and SECURITY_BULLETIN_EVENT." + default = [] +} + variable "deletion_protection" { type = bool description = "Whether or not to allow Terraform to destroy the cluster." @@ -707,13 +713,13 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration." default = [] } variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, APISERVER, CONTROLLER_MANAGER, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -729,12 +735,24 @@ variable "config_connector" { default = false } +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + variable "enable_l4_ilb_subsetting" { type = bool description = "Enable L4 ILB Subsetting on the cluster" default = false } +variable "enable_identity_service" { + type = bool + description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string