diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index c8bab6c35..7fc276c8a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -29,14 +29,14 @@ steps: - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage init --verbose'] -- id: converge node-pool-local +- id: apply node-pool-local waitFor: - init node-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage apply --verbose'] - id: verify node-pool-local waitFor: - - converge node-pool-local + - apply node-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage verify --verbose'] - id: destroy node-pool-local @@ -44,157 +44,6 @@ steps: - verify node-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage destroy --verbose'] -- id: apply sandbox-enabled-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage apply --verbose'] -- id: verify sandbox-enabled-local - waitFor: - - apply sandbox-enabled-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage verify --verbose'] -- id: destroy sandbox-enabled-local - waitFor: - - verify sandbox-enabled-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage destroy --verbose'] -- id: apply workload-identity-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage apply --verbose'] -- id: verify workload-identity-local - waitFor: - - apply workload-identity-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage verify --verbose'] -- id: destroy workload-identity-local - waitFor: - - verify workload-identity-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage teardown --verbose'] -- id: apply safer-cluster-iap-bastion-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage apply --verbose'] -- id: verify safer-cluster-iap-bastion-local - waitFor: - - apply safer-cluster-iap-bastion-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage verify --verbose'] -- id: destroy safer-cluster-iap-bastion-local - waitFor: - - verify safer-cluster-iap-bastion-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage teardown --verbose'] -- id: apply simple-zonal-with-asm-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage apply --verbose'] -- id: verify simple-zonal-with-asm-local - waitFor: - - apply simple-zonal-with-asm-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage verify --verbose'] -- id: destroy simple-zonal-with-asm-local - waitFor: - - verify simple-zonal-with-asm-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage teardown --verbose'] -- id: apply simple-autopilot-private-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage apply --verbose'] -- id: verify simple-autopilot-private-local - waitFor: - - apply simple-autopilot-private-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage verify --verbose'] -- id: destroy simple-autopilot-private-local - waitFor: - - verify simple-autopilot-private-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage teardown --verbose'] -- id: apply autopilot-private-firewalls - waitFor: - - create-all - - destroy simple-autopilot-private-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage apply --verbose'] -- id: verify autopilot-private-firewalls - waitFor: - - apply autopilot-private-firewalls - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage verify --verbose'] -- id: destroy autopilot-private-firewalls - waitFor: - - verify autopilot-private-firewalls - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage destroy --verbose'] -- id: apply simple-autopilot-public-local - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage apply --verbose'] -- id: verify simple-autopilot-public-local - waitFor: - - apply simple-autopilot-public-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage verify --verbose'] -- id: destroy simple-autopilot-public-local - waitFor: - - verify simple-autopilot-public-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage teardown --verbose'] -- id: apply private-zonal-with-networking - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage apply --verbose --test-dir test/integration'] -- id: verify private-zonal-with-networking - waitFor: - - apply private-zonal-with-networking - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage verify --verbose --test-dir test/integration'] -- id: teardown private-zonal-with-networking - waitFor: - - verify private-zonal-with-networking - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration'] -- id: apply simple-autopilot-private-non-default-sa - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage apply --verbose'] -- id: verify simple-autopilot-private-non-default-sa - waitFor: - - apply simple-autopilot-private-non-default-sa - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage verify --verbose'] -- id: teardown simple-autopilot-private-non-default-sa - waitFor: - - verify simple-autopilot-private-non-default-sa - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose'] -- id: apply simple-fleet-app-operator-permissions - waitFor: - - create-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage apply --verbose'] -- id: verify simple-fleet-app-operator-permissions - waitFor: - - apply simple-fleet-app-operator-permissions - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage verify --verbose'] -- id: teardown simple-fleet-app-operator-permissions - waitFor: - - verify simple-fleet-app-operator-permissions - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/test/integration/go.mod b/test/integration/go.mod index 675ea7627..94b4e3899 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -9,6 +9,7 @@ require ( github.com/gruntwork-io/terratest v0.48.1 github.com/hashicorp/terraform-json v0.24.0 github.com/stretchr/testify v1.10.0 + github.com/tidwall/gjson v1.18.0 ) require ( @@ -103,7 +104,6 @@ require ( github.com/pquerna/otp v1.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect diff --git a/test/integration/node_pool/node_pool_test.go b/test/integration/node_pool/node_pool_test.go index 92d6caaf5..cd76b9fb2 100644 --- a/test/integration/node_pool/node_pool_test.go +++ b/test/integration/node_pool/node_pool_test.go @@ -15,7 +15,6 @@ package node_pool import ( "fmt" - "slices" "testing" "time" @@ -23,8 +22,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" - "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" ) @@ -45,20 +42,14 @@ func TestNodePool(t *testing.T) { randomString := bpt.GetStringOutput("random_string") kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") - // Retrieve Project CAI + // // Retrieve Project CAI projectCAI := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster", "k8s.io/Node"})) - t.Log(projectCAI.Raw) - // Retrieve Cluster from CAI - clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) - if !projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data").Exists() { - t.Fatalf("Cluster not found: %s", clusterResourceName) - } - - cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data") - t.Log(cluster.Raw) + // // Retrieve Cluster from CAI + // clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) + // cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data") // Equivalent gcloud describe command (classic) - // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) // Cluster Assertions (classic) assert.Contains([]string{"RUNNING", "RECONCILING"}, cluster.Get("status").String(), "Cluster is Running") @@ -87,19 +78,8 @@ func TestNodePool(t *testing.T) { golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), ) - checkPaths := utils.GetTerminalJSONPaths(g.GetJSON()) - - exemptPaths := []string{"nodePools"} - checkPaths = slices.DeleteFunc(checkPaths, func(s string) bool { - return slices.Contains(exemptPaths, s) - }) - g.JSONPathEqs(assert, cluster, checkPaths) - // NodePool Assertions - nodePools := []string{"pool-01", "pool-02", "pool-03", "pool-04", "pool-05"} - for _, nodePool := range nodePools { - g.JSONPathEqs(assert, cluster.Get(fmt.Sprintf("nodePools.#(name==%s).name", nodePool)), utils.GetTerminalJSONPaths(g.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s).name", nodePool)))) - } + testutils.TGKEAssertGolden(assert, g, &cluster, []string{"pool-01", "pool-02", "pool-03", "pool-04", "pool-05"}, []string{"monitoringConfig.componentConfig.enableComponents"}) // nodePool-01 Assertions assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists") @@ -161,11 +141,12 @@ func TestNodePool(t *testing.T) { assert.True(cluster.Get("nodePools.#(name==\"pool-05\").config.advancedMachineFeatures.enableNestedVirtualization").Bool(), "has enable_nested_virtualization enabled") // K8s Assertions - gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) - k8sOpts := k8s.KubectlOptions{} - clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json") - assert.NoError(err) - clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp) + // gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId) + // k8sOpts := k8s.KubectlOptions{} + // clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json") + // assert.NoError(err) + // clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp) + assert.JSONEq(`[ { "effect": "PreferNoSchedule", @@ -178,7 +159,7 @@ func TestNodePool(t *testing.T) { "value": "true" } ]`, - clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-01\").spec.taints").String(), "has the expected taints") + projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-01\").resource.data.spec.taints").String(), "has the expected taints") assert.JSONEq(`[ { "effect": "PreferNoSchedule", @@ -191,7 +172,7 @@ func TestNodePool(t *testing.T) { "value": "present" } ]`, - clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-02\").spec.taints").String(), "has the expected all-pools-example taint") + projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-02\").resource.data.spec.taints").String(), "has the expected all-pools-example taint") assert.JSONEq(`[ { "effect": "PreferNoSchedule", @@ -204,7 +185,7 @@ func TestNodePool(t *testing.T) { "value": "gvisor" } ]`, - clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-03\").spec.taints").String(), "has the expected all-pools-example taint") + projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-03\").resource.data.spec.taints").String(), "has the expected all-pools-example taint") }) bpt.Test() diff --git a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go index aa912771d..d57745112 100644 --- a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go +++ b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go @@ -35,20 +35,20 @@ func TestSaferClusterIapBastion(t *testing.T) { // bpt.DefaultVerify(assert) testutils.TGKEVerify(t, bpt, assert) // Verify Resources - test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ") + testCommand, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ") // pre run ssh command so that ssh-keygen can run - gcloud.RunCmd(t, test_command, + gcloud.RunCmd(t, testCommand, gcloud.WithCommonArgs([]string{}), ) - cluster_version := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version")) + clusterVersion := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version")) - op := gcloud.Run(t, test_command, + op := gcloud.Run(t, testCommand, gcloud.WithCommonArgs([]string{}), ) - assert.Equal(cluster_version, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE") + assert.Equal(clusterVersion, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE") }) bpt.Test() diff --git a/test/integration/testutils/utils.go b/test/integration/testutils/utils.go index 1554678b9..38b21f995 100644 --- a/test/integration/testutils/utils.go +++ b/test/integration/testutils/utils.go @@ -15,14 +15,19 @@ package testutils import ( + "fmt" "slices" "strings" "testing" "time" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" + "golang.org/x/sync/errgroup" ) var ( @@ -36,6 +41,8 @@ var ( // API Rate limit exceeded errors can be retried. ".*rateLimitExceeded.*": "Rate limit exceeded.", } + + ClusterAlwaysExemptPaths = []string{"nodePools"} // node pools are checked by name ) func GetTestProjectFromSetup(t *testing.T, idx int) string { @@ -67,3 +74,43 @@ func TGKEVerifyExemptResources(t *testing.T, b *tft.TFBlueprintTest, assert *ass assert.Equal(tfjson.Actions{tfjson.ActionNoop}, r.Change.Actions, "Plan must be no-op for resource: %s", r.Address) } } + +// TGKEAssertGolden asserts a cluster and listed node pools against paths in golden image +func TGKEAssertGolden(assert *assert.Assertions, golden *golden.GoldenFile, clusterJson *gjson.Result, nodePools []string, exemptClusterPaths []string) { + // Retrieve golden paths + clusterCheckPaths := utils.GetTerminalJSONPaths(golden.GetJSON()) + + // Remove exempt cluster paths + exemptPaths := slices.Concat(exemptClusterPaths, ClusterAlwaysExemptPaths) + clusterCheckPaths = slices.DeleteFunc(clusterCheckPaths, func(s string) bool { + for _, exempPath := range exemptPaths { + if strings.HasPrefix(s, exempPath) { + return true + } + } + return false + }) + + // Cluster assertions + golden.JSONPathEqs(assert, *clusterJson, clusterCheckPaths) + + // NodePool assertions + for _, nodePool := range nodePools { + nodePool := nodePool + assert.Truef(clusterJson.Get(fmt.Sprintf("nodePools.#(name==%s).name", nodePool)).Exists(), "NodePool not found: %s", nodePool) + + nodeCheckPaths := utils.GetTerminalJSONPaths(golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool))) + + syncGroup := new(errgroup.Group) + syncGroup.SetLimit(24) + for _, nodeCheckPath := range nodeCheckPaths { + nodeCheckPath := nodeCheckPath + gotData := golden.ApplySanitizers(clusterJson.Get(fmt.Sprintf("#(nodePools.name==%s)", nodePool)).Get(nodeCheckPath).String()) + gfData := golden.GetJSON().Get(fmt.Sprintf("#(nodePools.name==%s)", nodePool)).Get(nodeCheckPath).String() + syncGroup.Go(func() error { + assert.Equalf(gfData, gotData, "For node %s path %q expected %q to match fixture %q", nodePool, nodeCheckPath, gotData, gfData) + return nil + }) + } + } +}