diff --git a/test/fixtures/node_pool/outputs.tf b/test/fixtures/node_pool/outputs.tf index a62317bf64..74103ff0be 100644 --- a/test/fixtures/node_pool/outputs.tf +++ b/test/fixtures/node_pool/outputs.tf @@ -83,3 +83,7 @@ output "service_account" { output "registry_project_ids" { value = var.registry_project_ids } + +output "random_string" { + value = random_string.suffix.result +} diff --git a/test/integration/go.mod b/test/integration/go.mod index dd87f81c69..10e060a197 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -9,7 +9,6 @@ require ( github.com/gruntwork-io/terratest v0.47.2 github.com/hashicorp/terraform-json v0.23.0 github.com/stretchr/testify v1.10.0 - github.com/tidwall/gjson v1.18.0 ) require ( @@ -52,7 +51,7 @@ require ( github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect github.com/hashicorp/hcl/v2 v2.20.1 // indirect - github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2 // indirect + github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -72,6 +71,7 @@ require ( github.com/pquerna/otp v1.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect @@ -85,7 +85,7 @@ require ( golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.4.0 // indirect + golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/test/integration/go.sum b/test/integration/go.sum index ab77546d08..ce2d8fab9c 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -189,6 +189,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.0 h1:EcxvASC45eOFrkk+cw9vP8Wztms3ZMWAQyBNEaI5MdU= github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.0/go.mod h1:homk/qeh6p4f7onHkLc5Vy8nG31ZmDK2pamBdHrc1OM= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.1 h1:BlnX8WYaNBRZ1ui3Rv1sgG6hwrVOjmAci1rpBeqFcMI= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.1/go.mod h1:gEV+whUF+NHOV6f6Hwrn64VEijQ2nOwNgpdotVm3rvM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -400,8 +402,8 @@ github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+Db github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= -github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2 h1:h+wcDC5wTiM+5YKjuLtUSWJtdtMVJNCRCPHoWieLpM4= -github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= +github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 h1:146llE+6P/9YO8RcHRehzGNiS9+OoirKW9/aML6/JIA= +github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI= github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -664,8 +666,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/test/integration/node_pool/node_pool_test.go b/test/integration/node_pool/node_pool_test.go index 9aae6f5143..fb653185cf 100644 --- a/test/integration/node_pool/node_pool_test.go +++ b/test/integration/node_pool/node_pool_test.go @@ -15,15 +15,20 @@ package node_pool import ( "fmt" + "slices" + "strings" "testing" "time" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" + "golang.org/x/sync/errgroup" ) func TestNodePool(t *testing.T) { @@ -34,20 +39,25 @@ func TestNodePool(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location") clusterName := bpt.GetStringOutput("cluster_name") + randomString := bpt.GetStringOutput("random_string") + kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") + //serviceAccount := bpt.GetStringOutput("service_account") - //cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + // Retrieve Cluster using CAI clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) - cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetType("container.googleapis.com/Cluster")).Get("#(name=\"" + clusterResourceName + "\").resource.data") + cluster := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data") + // Equivalent gcloud describe command + // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) // Cluster assert.Contains([]string{"RUNNING", "RECONCILING"}, cluster.Get("status").String(), "Cluster is Running") assert.Equal("COS_CONTAINERD", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.imageType").String(), "has the expected image type") - assert.Equal("[\n \"https://www.googleapis.com/auth/cloud-platform\"\n ]", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes").String(), "has the expected oauth scopes") + assert.Equal("https://www.googleapis.com/auth/cloud-platform", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0").String(), "has the expected oauth scopes") assert.Equal("default", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.serviceAccount").String(), "has the expected service account") assert.Equal("OPTIMIZE_UTILIZATION", cluster.Get("autoscaling.autoscalingProfile").String(), "has the expected autoscaling profile") assert.True(cluster.Get("autoscaling.enableNodeAutoprovisioning").Bool(), "has the expected node autoprovisioning") @@ -65,6 +75,84 @@ func TestNodePool(t *testing.T) { ]`, cluster.Get("autoscaling.resourceLimits").String(), "has the expected resource limits") + // Cluster (using golden image with sanitizer) + g := golden.NewOrUpdate(t, cluster.String(), + //golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(location, "LOCATION")), + //golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), + golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), + ) + + fmt.Println("START JSONEq") + validateJSONPaths := []string{ + "autoscaling.autoprovisioningNodePoolDefaults.imageType", + "autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0", + "autoscaling.autoprovisioningNodePoolDefaults.serviceAccount", + "autoscaling.autoscalingProfile", + "autoscaling.enableNodeAutoprovisioning", + "autoscaling.resourceLimits[0].maximum", + "autoscaling.resourceLimits[0].minimum", + "autoscaling.resourceLimits[0].resourceType", + "autoscaling.resourceLimits[1].maximum", + "autoscaling.resourceLimits[1].minimum", + "autoscaling.resourceLimits[1].resourceType", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, cluster, pth) + } + fmt.Println("END JSONEq") + + fmt.Println("START one path") + g.JSONPathEqs(assert, cluster, []string{"autoscaling.autoprovisioningNodePoolDefaults.imageType"}) + fmt.Println("END one path") + + fmt.Println("START multi path") + g.JSONPathEqs(assert, cluster, validateJSONPaths) + fmt.Println("END multi path") + + fmt.Println("START all paths") + // Test validating all paths in golden image + jsonPaths := utils.GetTerminalJSONPaths(g.GetJSON()) + + // List of paths exempt from validation + exemptJSONPathPrefixes := []string{ + "nodePools", // nodePools are unordered + "monitoringConfig.componentConfig.enableComponents", + } + + // Remove exempt paths by prefix + jsonPaths = slices.DeleteFunc(jsonPaths, func(s string) bool { + for _, path := range exemptJSONPathPrefixes { + if strings.HasPrefix(s, path) { + // prefix match + return true + } + } + // no prefix match + return false + }) + + jsonPaths = append(jsonPaths, "monitoringConfig.componentConfig.enableComponents") + // TODO replace with g.JSONPathEqs(assert, cluster, jsonPaths) in bpt v0.17.2 + + // JSONPathEqs asserts that json content in jsonPaths for got and goldenfile are the same + syncGroup := new(errgroup.Group) + syncGroup.SetLimit(24) + t.Logf("Checking %d JSON paths with max %d goroutines", len(jsonPaths), 20) + for _, jsonPath := range jsonPaths { + jsonPath := jsonPath + syncGroup.Go(func() error { + g.JSONEq(assert, cluster, jsonPath) + return nil + }) + } + if err := syncGroup.Wait(); err != nil { + t.Fatal(err) + } + fmt.Println("END all paths") + // Pool-01 assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists") assert.Equal("e2-medium", cluster.Get("nodePools.#(name==\"pool-01\").config.machineType").String(), "is the expected machine type") @@ -129,7 +217,7 @@ func TestNodePool(t *testing.T) { k8sOpts := k8s.KubectlOptions{} clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json") assert.NoError(err) - clusterNodes := testutils.ParseKubectlJSONResult(t, clusterNodesOp) + clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp) assert.JSONEq(`[ { "effect": "PreferNoSchedule", @@ -148,6 +236,11 @@ func TestNodePool(t *testing.T) { "effect": "PreferNoSchedule", "key": "all-pools-example", "value": "true" + }, + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "value": "present" } ]`, clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-02\").spec.taints").String(), "has the expected all-pools-example taint") @@ -156,6 +249,11 @@ func TestNodePool(t *testing.T) { "effect": "PreferNoSchedule", "key": "all-pools-example", "value": "true" + }, + { + "effect": "NoSchedule", + "key": "sandbox.gke.io/runtime", + "value": "gvisor" } ]`, clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-03\").spec.taints").String(), "has the expected all-pools-example taint") diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json new file mode 100644 index 0000000000..b628d90cbe --- /dev/null +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -0,0 +1,819 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoprovisioningNodePoolDefaults": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "management": { + "autoRepair": true, + "autoUpgrade": true, + "upgradeOptions": {} + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "upgradeSettings": { + "strategy": "SURGE" + } + }, + "autoscalingProfile": "OPTIMIZE_UTILIZATION", + "enableNodeAutoprovisioning": true, + "resourceLimits": [ + { + "maximum": "20", + "minimum": "5", + "resourceType": "cpu" + }, + { + "maximum": "30", + "minimum": "10", + "resourceType": "memory" + } + ] + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": false + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + } + }, + "currentMasterVersion": "1.30.5-gke.1443001", + "currentNodeCount": 8, + "currentNodeVersion": "1.30.5-gke.1443001", + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "KUBERNETES_ENDPOINT", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "identityServiceConfig": {}, + "initialClusterVersion": "1.30.5-gke.1443001", + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-RANDOM_STRING", + "defaultPodIpv4RangeUtilization": 0.0624, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-RANDOM_STRING", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "LOCATION", + "locations": [ + "LOCATION-b" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {} + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "CADVISOR", + "KUBELET" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "node-pool-cluster-RANDOM_STRING", + "network": "cft-gke-test-RANDOM_STRING", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-RANDOM_STRING", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/LOCATION/subnetworks/cft-gke-test-RANDOM_STRING" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-RANDOM_STRING", + "gke-node-pool-cluster-RANDOM_STRING-default-pool", + "all-node-example", + "pool-01-example" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "nodePools": [ + { + "autoscaling": {}, + "config": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-default-pool", + "all-node-example", + "pool-01-example" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-pool", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-RANDOM_STRING/nodePools/default-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 2, + "minNodeCount": 1 + }, + "config": { + "accelerators": [ + { + "acceleratorCount": "1", + "acceleratorType": "nvidia-tesla-p4", + "gpuDriverInstallationConfig": { + "gpuDriverVersion": "DEFAULT" + } + } + ], + "diskSizeGb": 30, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-02" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-02" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-02", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "nvidia.com/gpu", + "value": "present" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "initialNodeCount": 1, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-02", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-02", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 2, + "minNodeCount": 1 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-01", + "pool-01-example": "true" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000", + "net.core.rmem_max": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-01", + "shutdown-script": "kubectl --kubeconfig=/var/lib/kubelet/kubeconfig drain --force=true --ignore-daemonsets=true --delete-local-data \"$HOSTNAME\"" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-01", + "all-node-example", + "pool-01-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "PREFER_NO_SCHEDULE", + "key": "pool-01-example", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "initialNodeCount": 1, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-01", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-01", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "ANY", + "maxNodeCount": 100 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-04" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-04" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-04", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "cloud.google.com/gke-queued", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-04", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "podIpv4CidrSize": 24, + "queuedProvisioning": { + "enabled": true + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-04", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "advancedMachineFeatures": { + "enableNestedVirtualization": true + }, + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-05" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-05" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-05", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "initialNodeCount": 1, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-05", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-05", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "cpuCfsQuota": true, + "cpuManagerPolicy": "static", + "insecureKubeletReadonlyPortEnabled": false, + "podPidsLimit": "4096" + }, + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-03", + "sandbox.gke.io/runtime": "gvisor" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "20000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-03" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "sandboxConfig": { + "type": "GVISOR" + }, + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-03", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "sandbox.gke.io/runtime", + "value": "gvisor" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "initialNodeCount": 2, + "locations": [ + "LOCATION-b", + "LOCATION-c" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-03", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "172.16.0.0/18", + "podIpv4RangeUtilization": 0.0625, + "podRange": "test" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-03", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "autoprovisioned": true, + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "locations": [ + "LOCATION-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true, + "upgradeOptions": {} + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "nap-e2-medium-1d469r1p", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-RANDOM_STRING" + }, + "placementPolicy": {}, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/nap-e2-medium-1d469r1p", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "KUBERNETES_ENDPOINT" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-RANDOM_STRING", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-RANDOM_STRING", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "LOCATION" +} + diff --git a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go index 2e46f573f9..aa912771dd 100644 --- a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go +++ b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go @@ -23,7 +23,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" ) func TestSaferClusterIapBastion(t *testing.T) { @@ -34,7 +33,7 @@ func TestSaferClusterIapBastion(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ") diff --git a/test/integration/testutils/cai.go b/test/integration/testutils/cai.go deleted file mode 100644 index 69f819d67d..0000000000 --- a/test/integration/testutils/cai.go +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package cai provides a set of helpers to interact with Cloud Asset Inventory -package utils - -import ( - "testing" - "time" - - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" - "github.com/tidwall/gjson" -) - -type CmdCfg struct { - sleep int // minutes to sleep prior to CAI retreval. default: 2 - assetType string // asset type to retrieve. default: all -} - -type cmdOption func(*CmdCfg) - -// newCmdConfig sets defaults and options -func newCmdConfig(opts ...cmdOption) (*CmdCfg) { - caiOpts := &CmdCfg{ - sleep: 2, - assetType: "", - } - - for _, opt := range opts { - opt(caiOpts) - } - - return caiOpts -} - -// Set custom sleep minutes -func WithSleep(sleep int) cmdOption { - return func(f *CmdCfg) { - f.sleep = sleep - } -} - -// Set asset type -func WithAssetType(assetType string) cmdOption { - return func(f *CmdCfg) { - f.assetType = assetType - } -} - -// GetProjectResources returns the cloud asset inventory resources for a project as a gjson.Result -func GetProjectResources(t testing.TB, project string, opts ...cmdOption) gjson.Result { - caiOpts := newCmdConfig(opts...) - time.Sleep(time.Duration(caiOpts.sleep) * time.Minute) - if caiOpts.assetType != "" { - return gcloud.Runf(t, "asset list --project=%s --asset-types=%s --content-type=resource", project, caiOpts.assetType) - } else { - return gcloud.Runf(t, "asset list --project=%s --content-type=resource", project) - } -} diff --git a/test/integration/workload_identity/workload_identity_test.go b/test/integration/workload_identity/workload_identity_test.go index fe06e5321c..92ebc59541 100644 --- a/test/integration/workload_identity/workload_identity_test.go +++ b/test/integration/workload_identity/workload_identity_test.go @@ -22,7 +22,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" ) func TestWorkloadIdentity(t *testing.T) { @@ -33,7 +32,7 @@ func TestWorkloadIdentity(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location")