diff --git a/.changelog/4591.txt b/.changelog/4591.txt new file mode 100644 index 0000000000..123be36757 --- /dev/null +++ b/.changelog/4591.txt @@ -0,0 +1,4 @@ +```release-note:enhancement +container: add support for GKE Autopilot in `google_container_cluster` +container: promoted `networking_mode` to GA in `google_container_cluster` +``` diff --git a/google-beta/data_source_google_container_cluster_test.go b/google-beta/data_source_google_container_cluster_test.go index 8272320a34..24a4116b82 100644 --- a/google-beta/data_source_google_container_cluster_test.go +++ b/google-beta/data_source_google_container_cluster_test.go @@ -22,6 +22,7 @@ func TestAccContainerClusterDatasource_zonal(t *testing.T) { "google_container_cluster.kubes", // Remove once https://github.com/hashicorp/terraform/issues/21347 is fixed. map[string]struct{}{ + "enable_autopilot": {}, "enable_tpu": {}, "enable_binary_authorization": {}, "pod_security_policy_config.#": {}, @@ -48,6 +49,7 @@ func TestAccContainerClusterDatasource_regional(t *testing.T) { "google_container_cluster.kubes", // Remove once https://github.com/hashicorp/terraform/issues/21347 is fixed. map[string]struct{}{ + "enable_autopilot": {}, "enable_tpu": {}, "enable_binary_authorization": {}, "pod_security_policy_config.#": {}, diff --git a/google-beta/resource_container_cluster.go b/google-beta/resource_container_cluster.go index ed15f13976..b1fded77d4 100644 --- a/google-beta/resource_container_cluster.go +++ b/google-beta/resource_container_cluster.go @@ -122,6 +122,7 @@ func resourceContainerCluster() *schema.Resource { resourceNodeConfigEmptyGuestAccelerator, containerClusterPrivateClusterConfigCustomDiff, customdiff.ForceNewIfChange("enable_l4_ilb_subsetting", isBeenEnabled), + containerClusterAutopilotCustomizeDiff, ), Timeouts: &schema.ResourceTimeout{ @@ -229,12 +230,13 @@ func resourceContainerCluster() *schema.Resource { }, }, "network_policy_config": { - Type: schema.TypeList, - Optional: true, - Computed: true, - AtLeastOneOf: addonsConfigKeys, - MaxItems: 1, - Description: `Whether we should enable the network policy addon for the master. This must be enabled in order to enable network policy for the nodes. To enable this, you must also define a network_policy block, otherwise nothing will happen. It can only be disabled if the nodes already do not have network policies enabled. Defaults to disabled; set disabled = false to enable.`, + Type: schema.TypeList, + Optional: true, + Computed: true, + AtLeastOneOf: addonsConfigKeys, + MaxItems: 1, + Description: `Whether we should enable the network policy addon for the master. This must be enabled in order to enable network policy for the nodes. To enable this, you must also define a network_policy block, otherwise nothing will happen. It can only be disabled if the nodes already do not have network policies enabled. Defaults to disabled; set disabled = false to enable.`, + ConflictsWith: []string{"enable_autopilot"}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "disabled": { @@ -363,9 +365,10 @@ func resourceContainerCluster() *schema.Resource { MaxItems: 1, // This field is Optional + Computed because we automatically set the // enabled value to false if the block is not returned in API responses. - Optional: true, - Computed: true, - Description: `Per-cluster configuration of Node Auto-Provisioning with Cluster Autoscaler to automatically adjust the size of the cluster and create/delete node pools based on the current needs of the cluster's workload. See the guide to using Node Auto-Provisioning for more details.`, + Optional: true, + Computed: true, + Description: `Per-cluster configuration of Node Auto-Provisioning with Cluster Autoscaler to automatically adjust the size of the cluster and create/delete node pools based on the current needs of the cluster's workload. See the guide to using Node Auto-Provisioning for more details.`, + ConflictsWith: []string{"enable_autopilot"}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "enabled": { @@ -457,10 +460,11 @@ func resourceContainerCluster() *schema.Resource { }, "enable_binary_authorization": { - Default: false, - Type: schema.TypeBool, - Optional: true, - Description: `Enable Binary Authorization for this cluster. If enabled, all container images will be validated by Google Binary Authorization.`, + Default: false, + Type: schema.TypeBool, + Optional: true, + Description: `Enable Binary Authorization for this cluster. If enabled, all container images will be validated by Google Binary Authorization.`, + ConflictsWith: []string{"enable_autopilot"}, }, "enable_kubernetes_alpha": { @@ -487,19 +491,29 @@ func resourceContainerCluster() *schema.Resource { }, "enable_shielded_nodes": { + Type: schema.TypeBool, + Optional: true, + Computed: true, + Description: `Enable Shielded Nodes features on all nodes in this cluster.`, + ConflictsWith: []string{"enable_autopilot"}, + }, + + "enable_autopilot": { Type: schema.TypeBool, Optional: true, - Default: false, - Description: `Enable Shielded Nodes features on all nodes in this cluster. Defaults to false.`, + ForceNew: true, + Description: `Enable Autopilot for this cluster.`, + // ConflictsWith: many fields, see https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview#comparison. The conflict is only set one-way, on other fields w/ this field. }, "authenticator_groups_config": { - Type: schema.TypeList, - Optional: true, - Computed: true, - ForceNew: true, - MaxItems: 1, - Description: `Configuration for the Google Groups for GKE feature.`, + Type: schema.TypeList, + Optional: true, + Computed: true, + ForceNew: true, + MaxItems: 1, + Description: `Configuration for the Google Groups for GKE feature.`, + ConflictsWith: []string{"enable_autopilot"}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "security_group": { @@ -769,11 +783,12 @@ func resourceContainerCluster() *schema.Resource { }, "network_policy": { - Type: schema.TypeList, - Optional: true, - Computed: true, - MaxItems: 1, - Description: `Configuration options for the NetworkPolicy feature.`, + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Description: `Configuration options for the NetworkPolicy feature.`, + ConflictsWith: []string{"enable_autopilot"}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "enabled": { @@ -803,7 +818,8 @@ func resourceContainerCluster() *schema.Resource { Elem: &schema.Resource{ Schema: schemaNodePool, }, - Description: `List of node pools associated with this cluster. See google_container_node_pool for schema. Warning: node pools defined inside a cluster can't be changed (or added/removed) after cluster creation without deleting and recreating the entire cluster. Unless you absolutely need the ability to say "these are the only node pools associated with this cluster", use the google_container_node_pool resource instead of this property.`, + Description: `List of node pools associated with this cluster. See google_container_node_pool for schema. Warning: node pools defined inside a cluster can't be changed (or added/removed) after cluster creation without deleting and recreating the entire cluster. Unless you absolutely need the ability to say "these are the only node pools associated with this cluster", use the google_container_node_pool resource instead of this property.`, + ConflictsWith: []string{"enable_autopilot"}, }, "node_version": { @@ -882,6 +898,7 @@ func resourceContainerCluster() *schema.Resource { Type: schema.TypeList, MaxItems: 1, ForceNew: true, + Computed: true, Optional: true, ConflictsWith: []string{"cluster_ipv4_cidr"}, Description: `Configuration of cluster IP allocation for VPC-native clusters. Adding this block enables IP aliasing, making the cluster VPC-native instead of routes-based.`, @@ -940,9 +957,10 @@ func resourceContainerCluster() *schema.Resource { }, "remove_default_node_pool": { - Type: schema.TypeBool, - Optional: true, - Description: `If true, deletes the default node pool upon cluster creation. If you're using google_container_node_pool resources with no default node pool, this should be set to true, alongside setting initial_node_count to at least 1.`, + Type: schema.TypeBool, + Optional: true, + Description: `If true, deletes the default node pool upon cluster creation. If you're using google_container_node_pool resources with no default node pool, this should be set to true, alongside setting initial_node_count to at least 1.`, + ConflictsWith: []string{"enable_autopilot"}, }, "private_cluster_config": { @@ -1024,11 +1042,12 @@ func resourceContainerCluster() *schema.Resource { }, "default_max_pods_per_node": { - Type: schema.TypeInt, - Optional: true, - ForceNew: true, - Computed: true, - Description: `The default maximum number of pods per node in this cluster. This doesn't work on "routes-based" clusters, clusters that don't have IP Aliasing enabled.`, + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Computed: true, + Description: `The default maximum number of pods per node in this cluster. This doesn't work on "routes-based" clusters, clusters that don't have IP Aliasing enabled.`, + ConflictsWith: []string{"enable_autopilot"}, }, "vertical_pod_autoscaling": { @@ -1047,10 +1066,12 @@ func resourceContainerCluster() *schema.Resource { }, }, "workload_identity_config": { - Type: schema.TypeList, - MaxItems: 1, - Optional: true, - Description: `Configuration for the use of Kubernetes Service Accounts in GCP IAM policies.`, + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Computed: true, + Description: `Configuration for the use of Kubernetes Service Accounts in GCP IAM policies.`, + ConflictsWith: []string{"enable_autopilot"}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "identity_namespace": { @@ -1158,10 +1179,11 @@ func resourceContainerCluster() *schema.Resource { }, "enable_intranode_visibility": { - Type: schema.TypeBool, - Optional: true, - Description: `Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network.`, - Default: false, + Type: schema.TypeBool, + Optional: true, + Computed: true, + Description: `Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network.`, + ConflictsWith: []string{"enable_autopilot"}, }, "enable_l4_ilb_subsetting": { Type: schema.TypeBool, @@ -1322,8 +1344,8 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er Enabled: d.Get("enable_binary_authorization").(bool), ForceSendFields: []string{"Enabled"}, }, - ShieldedNodes: &containerBeta.ShieldedNodes{ - Enabled: d.Get("enable_shielded_nodes").(bool), + Autopilot: &containerBeta.Autopilot{ + Enabled: d.Get("enable_autopilot").(bool), ForceSendFields: []string{"Enabled"}, }, ReleaseChannel: expandReleaseChannel(d.Get("release_channel")), @@ -1342,6 +1364,13 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er ResourceLabels: expandStringMap(d, "resource_labels"), } + if v, ok := d.GetOk("enable_shielded_nodes"); ok { + cluster.ShieldedNodes = &containerBeta.ShieldedNodes{ + Enabled: v.(bool), + ForceSendFields: []string{"Enabled"}, + } + } + if v, ok := d.GetOk("default_max_pods_per_node"); ok { cluster.DefaultMaxPodsConstraint = expandDefaultMaxPodsConstraint(v) } @@ -1651,6 +1680,11 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro if err := d.Set("enable_binary_authorization", cluster.BinaryAuthorization != nil && cluster.BinaryAuthorization.Enabled); err != nil { return fmt.Errorf("Error setting enable_binary_authorization: %s", err) } + if cluster.Autopilot != nil { + if err := d.Set("enable_autopilot", cluster.Autopilot.Enabled); err != nil { + return fmt.Errorf("Error setting enable_autopilot: %s", err) + } + } if cluster.ShieldedNodes != nil { if err := d.Set("enable_shielded_nodes", cluster.ShieldedNodes.Enabled); err != nil { return fmt.Errorf("Error setting enable_shielded_nodes: %s", err) @@ -3021,6 +3055,9 @@ func expandMaintenancePolicy(d *schema.ResourceData, meta interface{}) *containe func expandClusterAutoscaling(configured interface{}, d *schema.ResourceData) *containerBeta.ClusterAutoscaling { l, ok := configured.([]interface{}) if !ok || l == nil || len(l) == 0 || l[0] == nil { + if v, ok := d.GetOk("enable_autopilot"); ok && v == true { + return nil + } return &containerBeta.ClusterAutoscaling{ EnableNodeAutoprovisioning: false, ForceSendFields: []string{"EnableNodeAutoprovisioning"}, @@ -3936,6 +3973,20 @@ func containerClusterPrivateClusterConfigCustomDiff(_ context.Context, d *schema return nil } +// Autopilot clusters have preconfigured defaults: https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview#comparison. +// This function modifies the diff so users can see what these will be during plan time. +func containerClusterAutopilotCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error { + if d.HasChange("enable_autopilot") && d.Get("enable_autopilot").(bool) { + if err := d.SetNew("enable_intranode_visibility", true); err != nil { + return err + } + if err := d.SetNew("enable_shielded_nodes", true); err != nil { + return err + } + } + return nil +} + func podSecurityPolicyCfgSuppress(k, old, new string, r *schema.ResourceData) bool { if k == "pod_security_policy_config.#" && old == "1" && new == "0" { if v, ok := r.GetOk("pod_security_policy_config"); ok { diff --git a/google-beta/resource_container_cluster_test.go b/google-beta/resource_container_cluster_test.go index 9ad85259a1..d290d86fb2 100644 --- a/google-beta/resource_container_cluster_test.go +++ b/google-beta/resource_container_cluster_test.go @@ -1658,6 +1658,48 @@ func TestAccContainerCluster_withShieldedNodes(t *testing.T) { }) } +func TestAccContainerCluster_withAutopilot(t *testing.T) { + t.Parallel() + + containerNetName := fmt.Sprintf("tf-test-container-net-%s", randString(t, 10)) + clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withAutopilot(containerNetName, clusterName, true), + }, + { + ResourceName: "google_container_cluster.with_autopilot", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + +func TestAccContainerCluster_errorAutopilotLocation(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withInvalidAutopilotLocation(clusterName, "us-central1-a"), + ExpectError: regexp.MustCompile(`Autopilot clusters must be regional clusters.`), + }, + }, + }) +} + func TestAccContainerCluster_withWorkloadIdentityConfig(t *testing.T) { t.Parallel() @@ -4505,3 +4547,68 @@ resource "google_container_cluster" "primary" { } `, name) } + +func testAccContainerCluster_withAutopilot(containerNetName string, clusterName string, enabled bool) string { + return fmt.Sprintf(` +resource "google_compute_network" "container_network" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork" { + name = google_compute_network.container_network.name + network = google_compute_network.container_network.name + ip_cidr_range = "10.0.36.0/24" + region = "us-central1" + private_ip_google_access = true + + secondary_ip_range { + range_name = "pod" + ip_cidr_range = "10.0.0.0/19" + } + + secondary_ip_range { + range_name = "svc" + ip_cidr_range = "10.0.32.0/22" + } +} + +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "with_autopilot" { + name = "%s" + location = "us-central1" + enable_autopilot = %v + min_master_version = "latest" + release_channel { + channel = "RAPID" + } + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork.name + ip_allocation_policy { + cluster_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + services_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[1].range_name + } + addons_config { + horizontal_pod_autoscaling { + disabled = false + } + } + vertical_pod_autoscaling { + enabled = true + } +} +`, containerNetName, clusterName, enabled) +} + +func testAccContainerCluster_withInvalidAutopilotLocation(clusterName string, location string) string { + return fmt.Sprintf(` +resource "google_container_cluster" "with_invalid_location" { + name = "%s" + location = "%s" + enable_autopilot = true +} +`, clusterName, location) +} diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown index dccfb0af16..6f3f23c53a 100644 --- a/website/docs/r/container_cluster.html.markdown +++ b/website/docs/r/container_cluster.html.markdown @@ -159,6 +159,11 @@ for more information. * `enable_shielded_nodes` - (Optional) Enable Shielded Nodes features on all nodes in this cluster. Defaults to `false`. +* `enable_autopilot` - (Optional) Enable Autopilot for this cluster. Defaults to `false`. + Note that when this option is enabled, certain features of Standard GKE are not available. + See the [official documentation](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview#comparison) + for available features. + * `initial_node_count` - (Optional) The number of nodes to create in this cluster's default node pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Must be set if `node_pool` is not set. If you're using