From c2baed73999043038d308e059dbfae0f55fd576f Mon Sep 17 00:00:00 2001 From: Max Portocarrero CI&T <105444618+maxi-cit@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:27:22 -0500 Subject: [PATCH] Add resource manager tags to GKE autopilot (#10123) --- .../services/container/node_config.go.erb | 3 +- .../resource_container_cluster.go.erb | 30 ++ .../resource_container_cluster_test.go.erb | 468 ++++++++++++++++++ .../docs/r/container_cluster.html.markdown | 4 +- 4 files changed, 502 insertions(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.erb b/mmv1/third_party/terraform/services/container/node_config.go.erb index 089c5943aaea..3c60164f42c4 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.erb +++ b/mmv1/third_party/terraform/services/container/node_config.go.erb @@ -979,7 +979,7 @@ func expandNodeConfig(v interface{}) *container.NodeConfig { if v,ok := nodeConfig["enable_confidential_storage"]; ok { nc.EnableConfidentialStorage = v.(bool) } - + <% unless version == "ga" -%> if v, ok := nodeConfig["host_maintenance_policy"]; ok { nc.HostMaintenancePolicy = expandHostMaintenancePolicy(v) @@ -1249,7 +1249,6 @@ func flattenResourceManagerTags(c *container.ResourceManagerTags) map[string]int for k, v := range c.Tags { rmt[k] = v } - } return rmt diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb index 93f6dadd3693..221b01ac9403 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb @@ -1448,6 +1448,11 @@ func ResourceContainerCluster() *schema.Resource { }, }, }, + "resource_manager_tags": { + Type: schema.TypeMap, + Optional: true, + Description: `A map of resource manager tags. Resource manager tag keys and values have the same definition as resource manager tags. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456. The field is ignored (both PUT & PATCH) when empty.`, + }, }, }, }, @@ -4162,6 +4167,24 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er log.Printf("[INFO] GKE cluster %s node pool auto config network tags have been updated", d.Id()) } + if d.HasChange("node_pool_auto_config.0.resource_manager_tags") { + rmtags := d.Get("node_pool_auto_config.0.resource_manager_tags") + + req := &container.UpdateClusterRequest{ + Update: &container.ClusterUpdate{ + DesiredNodePoolAutoConfigResourceManagerTags: expandResourceManagerTags(rmtags), + }, + } + + updateF := updateFunc(req, "updating GKE cluster node pool auto config resource manager tags") + // Call update serially. + if err := transport_tpg.LockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s node pool auto config resource manager tags have been updated", d.Id()) + } + d.Partial(false) <% unless version == 'ga' -%> @@ -5426,6 +5449,10 @@ func expandNodePoolAutoConfig(configured interface{}) *container.NodePoolAutoCon npac.NetworkTags = expandNodePoolAutoConfigNetworkTags(v) } + if v, ok := config["resource_manager_tags"]; ok && len(v.(map[string]interface{})) > 0 { + npac.ResourceManagerTags = expandResourceManagerTags(v) + } + return npac } @@ -6234,6 +6261,9 @@ func flattenNodePoolAutoConfig(c *container.NodePoolAutoConfig) []map[string]int if c.NetworkTags != nil { result["network_tags"] = flattenNodePoolAutoConfigNetworkTags(c.NetworkTags) } + if c.ResourceManagerTags != nil { + result["resource_manager_tags"] = flattenResourceManagerTags(c.ResourceManagerTags) + } return []map[string]interface{}{result} } diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb index 31367bcbc07d..bf45123177b3 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb @@ -2980,6 +2980,62 @@ func TestAccContainerCluster_withAutopilotNetworkTags(t *testing.T) { }) } +func TestAccContainerCluster_withAutopilotResourceManagerTags(t *testing.T) { + t.Parallel() + + pid := envvar.GetTestProjectFromEnv() + + randomSuffix := acctest.RandString(t, 10) + clusterName := fmt.Sprintf("tf-test-cluster-%s", randomSuffix) + clusterNetName := fmt.Sprintf("tf-test-container-net-%s", randomSuffix) + clusterSubnetName := fmt.Sprintf("tf-test-container-subnet-%s", randomSuffix) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + ExternalProviders: map[string]resource.ExternalProvider{ + "time": {}, + }, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withAutopilotResourceManagerTags(pid, clusterName, clusterNetName, clusterSubnetName, randomSuffix), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttrSet("google_container_cluster.with_autopilot", "self_link"), + resource.TestCheckResourceAttrSet("google_container_cluster.with_autopilot", "node_pool_auto_config.0.resource_manager_tags.%"), + ), + }, + { + ResourceName: "google_container_cluster.with_autopilot", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + { + Config: testAccContainerCluster_withAutopilotResourceManagerTagsUpdate1(pid, clusterName, clusterNetName, clusterSubnetName, randomSuffix), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttrSet("google_container_cluster.with_autopilot", "node_pool_auto_config.0.resource_manager_tags.%"), + ), + }, + { + ResourceName: "google_container_cluster.with_autopilot", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + { + Config: testAccContainerCluster_withAutopilotResourceManagerTagsUpdate2(pid, clusterName, clusterNetName, clusterSubnetName, randomSuffix), + }, + { + ResourceName: "google_container_cluster.with_autopilot", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + }, + }) +} + func TestAccContainerCluster_withWorkloadIdentityConfig(t *testing.T) { t.Parallel() @@ -9735,3 +9791,415 @@ resource "google_container_cluster" "primary" { } `, projectID, randomSuffix, clusterName, networkName, subnetworkName) } + +func testAccContainerCluster_withAutopilotResourceManagerTags(projectID, clusterName, networkName, subnetworkName, randomSuffix string) string { + return fmt.Sprintf(` +data "google_project" "project" { + project_id = "%[1]s" +} + +resource "google_project_iam_binding" "tagHoldAdmin" { + project = "%[1]s" + role = "roles/resourcemanager.tagHoldAdmin" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + ] +} + +resource "google_project_iam_binding" "tagUser" { + project = "%[1]s" + role = "roles/resourcemanager.tagUser" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + "serviceAccount:${data.google_project.project.number}@cloudservices.gserviceaccount.com", + ] + + depends_on = [google_project_iam_binding.tagHoldAdmin] +} + +resource "time_sleep" "wait_120_seconds" { + create_duration = "120s" + + depends_on = [ + google_project_iam_binding.tagHoldAdmin, + google_project_iam_binding.tagUser + ] +} + +resource "google_tags_tag_key" "key1" { + parent = "projects/%[1]s" + short_name = "foobarbaz1-%[2]s" + description = "For foo/bar1 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [google_compute_network.container_network] +} + +resource "google_tags_tag_value" "value1" { + parent = "tagKeys/${google_tags_tag_key.key1.name}" + short_name = "foo1-%[2]s" + description = "For foo1 resources" +} + +resource "google_tags_tag_key" "key2" { + parent = "projects/%[1]s" + short_name = "foobarbaz2-%[2]s" + description = "For foo/bar2 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [ + google_compute_network.container_network, + google_tags_tag_key.key1 + ] +} + +resource "google_tags_tag_value" "value2" { + parent = "tagKeys/${google_tags_tag_key.key2.name}" + short_name = "foo2-%[2]s" + description = "For foo2 resources" +} + +resource "google_compute_network" "container_network" { + name = "%[4]s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork" { + name = "%[5]s" + network = google_compute_network.container_network.name + ip_cidr_range = "10.0.36.0/24" + region = "us-central1" + private_ip_google_access = true + + secondary_ip_range { + range_name = "pod" + ip_cidr_range = "10.0.0.0/19" + } + + secondary_ip_range { + range_name = "svc" + ip_cidr_range = "10.0.32.0/22" + } +} + +data "google_container_engine_versions" "uscentral1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "with_autopilot" { + name = "%[3]s" + location = "us-central1" + min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"] + enable_autopilot = true + + deletion_protection = false + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork.name + ip_allocation_policy { + cluster_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + services_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[1].range_name + } + + node_pool_auto_config { + resource_manager_tags = { + "tagKeys/${google_tags_tag_key.key1.name}" = "tagValues/${google_tags_tag_value.value1.name}" + } + } + + addons_config { + horizontal_pod_autoscaling { + disabled = false + } + } + vertical_pod_autoscaling { + enabled = true + } + + timeouts { + create = "30m" + update = "40m" + } + + depends_on = [time_sleep.wait_120_seconds] +} +`, projectID, randomSuffix, clusterName, networkName, subnetworkName) +} + +func testAccContainerCluster_withAutopilotResourceManagerTagsUpdate1(projectID, clusterName, networkName, subnetworkName, randomSuffix string) string { + return fmt.Sprintf(` +data "google_project" "project" { + project_id = "%[1]s" +} + +resource "google_project_iam_binding" "tagHoldAdmin" { + project = "%[1]s" + role = "roles/resourcemanager.tagHoldAdmin" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + ] +} + +resource "google_project_iam_binding" "tagUser" { + project = "%[1]s" + role = "roles/resourcemanager.tagUser" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + "serviceAccount:${data.google_project.project.number}@cloudservices.gserviceaccount.com", + ] + + depends_on = [google_project_iam_binding.tagHoldAdmin] +} + +resource "time_sleep" "wait_120_seconds" { + create_duration = "120s" + + depends_on = [ + google_project_iam_binding.tagHoldAdmin, + google_project_iam_binding.tagUser + ] +} + +resource "google_tags_tag_key" "key1" { + parent = "projects/%[1]s" + short_name = "foobarbaz1-%[2]s" + description = "For foo/bar1 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [google_compute_network.container_network] +} + +resource "google_tags_tag_value" "value1" { + parent = "tagKeys/${google_tags_tag_key.key1.name}" + short_name = "foo1-%[2]s" + description = "For foo1 resources" +} + +resource "google_tags_tag_key" "key2" { + parent = "projects/%[1]s" + short_name = "foobarbaz2-%[2]s" + description = "For foo/bar2 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [ + google_compute_network.container_network, + google_tags_tag_key.key1 + ] +} + +resource "google_tags_tag_value" "value2" { + parent = "tagKeys/${google_tags_tag_key.key2.name}" + short_name = "foo2-%[2]s" + description = "For foo2 resources" +} + +resource "google_compute_network" "container_network" { + name = "%[4]s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork" { + name = "%[5]s" + network = google_compute_network.container_network.name + ip_cidr_range = "10.0.36.0/24" + region = "us-central1" + private_ip_google_access = true + + secondary_ip_range { + range_name = "pod" + ip_cidr_range = "10.0.0.0/19" + } + + secondary_ip_range { + range_name = "svc" + ip_cidr_range = "10.0.32.0/22" + } +} + +data "google_container_engine_versions" "uscentral1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "with_autopilot" { + name = "%[3]s" + location = "us-central1" + min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"] + enable_autopilot = true + + deletion_protection = false + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork.name + ip_allocation_policy { + cluster_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + services_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[1].range_name + } + + node_pool_auto_config { + resource_manager_tags = { + "tagKeys/${google_tags_tag_key.key1.name}" = "tagValues/${google_tags_tag_value.value1.name}" + "tagKeys/${google_tags_tag_key.key2.name}" = "tagValues/${google_tags_tag_value.value2.name}" + } + } + + addons_config { + horizontal_pod_autoscaling { + disabled = false + } + } + vertical_pod_autoscaling { + enabled = true + } + + timeouts { + create = "30m" + update = "40m" + } + + depends_on = [time_sleep.wait_120_seconds] +} +`, projectID, randomSuffix, clusterName, networkName, subnetworkName) +} + +func testAccContainerCluster_withAutopilotResourceManagerTagsUpdate2(projectID, clusterName, networkName, subnetworkName, randomSuffix string) string { + return fmt.Sprintf(` +data "google_project" "project" { + project_id = "%[1]s" +} + +resource "google_project_iam_binding" "tagHoldAdmin" { + project = "%[1]s" + role = "roles/resourcemanager.tagHoldAdmin" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + ] +} + +resource "google_project_iam_binding" "tagUser" { + project = "%[1]s" + role = "roles/resourcemanager.tagUser" + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com", + "serviceAccount:${data.google_project.project.number}@cloudservices.gserviceaccount.com", + ] + + depends_on = [google_project_iam_binding.tagHoldAdmin] +} + +resource "time_sleep" "wait_120_seconds" { + create_duration = "120s" + + depends_on = [ + google_project_iam_binding.tagHoldAdmin, + google_project_iam_binding.tagUser + ] +} + +resource "google_tags_tag_key" "key1" { + parent = "projects/%[1]s" + short_name = "foobarbaz1-%[2]s" + description = "For foo/bar1 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [google_compute_network.container_network] +} + +resource "google_tags_tag_value" "value1" { + parent = "tagKeys/${google_tags_tag_key.key1.name}" + short_name = "foo1-%[2]s" + description = "For foo1 resources" +} + +resource "google_tags_tag_key" "key2" { + parent = "projects/%[1]s" + short_name = "foobarbaz2-%[2]s" + description = "For foo/bar2 resources" + purpose = "GCE_FIREWALL" + purpose_data = { + network = "%[1]s/%[4]s" + } + + depends_on = [ + google_compute_network.container_network, + google_tags_tag_key.key1 + ] +} + +resource "google_tags_tag_value" "value2" { + parent = "tagKeys/${google_tags_tag_key.key2.name}" + short_name = "foo2-%[2]s" + description = "For foo2 resources" +} + +resource "google_compute_network" "container_network" { + name = "%[4]s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork" { + name = "%[5]s" + network = google_compute_network.container_network.name + ip_cidr_range = "10.0.36.0/24" + region = "us-central1" + private_ip_google_access = true + + secondary_ip_range { + range_name = "pod" + ip_cidr_range = "10.0.0.0/19" + } + + secondary_ip_range { + range_name = "svc" + ip_cidr_range = "10.0.32.0/22" + } +} + +data "google_container_engine_versions" "uscentral1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "with_autopilot" { + name = "%[3]s" + location = "us-central1" + min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"] + enable_autopilot = true + + deletion_protection = false + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork.name + ip_allocation_policy { + cluster_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + services_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[1].range_name + } + + addons_config { + horizontal_pod_autoscaling { + disabled = false + } + } + vertical_pod_autoscaling { + enabled = true + } + + timeouts { + create = "30m" + update = "40m" + } + + depends_on = [time_sleep.wait_120_seconds] +} +`, projectID, randomSuffix, clusterName, networkName, subnetworkName) +} diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 54837be29211..2212db48beed 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -15,7 +15,7 @@ To get more information about GKE clusters, see: * [About cluster configuration choices](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters) * Terraform guidance * [Using GKE with Terraform](/docs/providers/google/guides/using_gke_with_terraform.html) - * [Provision a GKE Cluster (Google Cloud) Learn tutorial](https://learn.hashicorp.com/tutorials/terraform/gke?in=terraform/kubernetes&utm_source=WEBSITE&utm_medium=WEB_IO&utm_offer=ARTICLE_PAGE&utm_content=DOCS) + * [Provision a GKE Cluster (Google Cloud) Learn tutorial](https://learn.hashicorp.com/tutorials/terraform/gke?in=terraform/kubernetes&utm_source=WEBSITE&utm_medium=WEB_IO&utm_offer=ARTICLE_PAGE&utm_content=DOCS) -> On version 5.0.0+ of the provider, you must explicitly set `deletion_protection = false` and run `terraform apply` to write the field to state in order to destroy a cluster. @@ -1033,6 +1033,8 @@ workload_identity_config { The `node_pool_auto_config` block supports: +* `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`. + * `network_tags` (Optional) - The network tag config for the cluster's automatically provisioned node pools. The `network_tags` block supports: