Skip to content

Commit

Permalink
multi asset types
Browse files Browse the repository at this point in the history 
validate all paths
  • Loading branch information
@apeabody
apeabody committed Dec 7, 2024
1 parent 5084c25 commit 2df4b54
Show file tree
Hide file tree
Showing 8 changed files with 938 additions and 89 deletions.
4 changes: 4 additions & 0 deletions test/fixtures/node_pool/outputs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,7 @@ output "service_account" {
output "registry_project_ids" {
value = var.registry_project_ids
}

output "random_string" {
value = random_string.suffix.result
}
6 changes: 3 additions & 3 deletions test/integration/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ require (
github.com/gruntwork-io/terratest v0.47.2
github.com/hashicorp/terraform-json v0.23.0
github.com/stretchr/testify v1.10.0
github.com/tidwall/gjson v1.18.0
)

require (
Expand Down Expand Up @@ -52,7 +51,7 @@ require (
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect
github.com/hashicorp/hcl/v2 v2.20.1 // indirect
github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2 // indirect
github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
Expand All @@ -72,6 +71,7 @@ require (
github.com/pquerna/otp v1.4.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
Expand All @@ -85,7 +85,7 @@ require (
golang.org/x/mod v0.22.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
Expand Down
10 changes: 6 additions & 4 deletions test/integration/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.0 h1:EcxvASC45eOFrkk+cw9vP8Wztms3ZMWAQyBNEaI5MdU=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.0/go.mod h1:homk/qeh6p4f7onHkLc5Vy8nG31ZmDK2pamBdHrc1OM=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.1 h1:BlnX8WYaNBRZ1ui3Rv1sgG6hwrVOjmAci1rpBeqFcMI=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.1/go.mod h1:gEV+whUF+NHOV6f6Hwrn64VEijQ2nOwNgpdotVm3rvM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
Expand Down Expand Up @@ -400,8 +402,8 @@ github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+Db
github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc=
github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4=
github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2 h1:h+wcDC5wTiM+5YKjuLtUSWJtdtMVJNCRCPHoWieLpM4=
github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 h1:146llE+6P/9YO8RcHRehzGNiS9+OoirKW9/aML6/JIA=
github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI=
github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down Expand Up @@ -664,8 +666,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
110 changes: 104 additions & 6 deletions test/integration/node_pool/node_pool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,20 @@ package node_pool

import (
"fmt"
"slices"
"strings"
"testing"
"time"

"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/stretchr/testify/assert"
"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils"
"golang.org/x/sync/errgroup"
)

func TestNodePool(t *testing.T) {
Expand All @@ -34,20 +39,25 @@ func TestNodePool(t *testing.T) {
bpt.DefineVerify(func(assert *assert.Assertions) {
// Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token
// bpt.DefaultVerify(assert)
gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources
testutils.TGKEVerify(t, bpt, assert) // Verify Resources

projectId := bpt.GetStringOutput("project_id")
location := bpt.GetStringOutput("location")
clusterName := bpt.GetStringOutput("cluster_name")
randomString := bpt.GetStringOutput("random_string")
kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint")
//serviceAccount := bpt.GetStringOutput("service_account")

//cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
// Retrieve Cluster using CAI
clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName)
cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetType("container.googleapis.com/Cluster")).Get("#(name=\"" + clusterResourceName + "\").resource.data")
cluster := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data")
// Equivalent gcloud describe command
// cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)

// Cluster
assert.Contains([]string{"RUNNING", "RECONCILING"}, cluster.Get("status").String(), "Cluster is Running")
assert.Equal("COS_CONTAINERD", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.imageType").String(), "has the expected image type")
assert.Equal("[\n \"https://www.googleapis.com/auth/cloud-platform\"\n ]", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes").String(), "has the expected oauth scopes")
assert.Equal("https://www.googleapis.com/auth/cloud-platform", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0").String(), "has the expected oauth scopes")
assert.Equal("default", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.serviceAccount").String(), "has the expected service account")
assert.Equal("OPTIMIZE_UTILIZATION", cluster.Get("autoscaling.autoscalingProfile").String(), "has the expected autoscaling profile")
assert.True(cluster.Get("autoscaling.enableNodeAutoprovisioning").Bool(), "has the expected node autoprovisioning")
Expand All @@ -65,6 +75,84 @@ func TestNodePool(t *testing.T) {
]`,
cluster.Get("autoscaling.resourceLimits").String(), "has the expected resource limits")

// Cluster (using golden image with sanitizer)
g := golden.NewOrUpdate(t, cluster.String(),
//golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")),
golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")),
golden.WithSanitizer(golden.StringSanitizer(location, "LOCATION")),
//golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")),
golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")),
golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")),
)

fmt.Println("START JSONEq")
validateJSONPaths := []string{
"autoscaling.autoprovisioningNodePoolDefaults.imageType",
"autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0",
"autoscaling.autoprovisioningNodePoolDefaults.serviceAccount",
"autoscaling.autoscalingProfile",
"autoscaling.enableNodeAutoprovisioning",
"autoscaling.resourceLimits[0].maximum",
"autoscaling.resourceLimits[0].minimum",
"autoscaling.resourceLimits[0].resourceType",
"autoscaling.resourceLimits[1].maximum",
"autoscaling.resourceLimits[1].minimum",
"autoscaling.resourceLimits[1].resourceType",
}
for _, pth := range validateJSONPaths {
g.JSONEq(assert, cluster, pth)
}
fmt.Println("END JSONEq")

fmt.Println("START one path")
g.JSONPathEqs(assert, cluster, []string{"autoscaling.autoprovisioningNodePoolDefaults.imageType"})
fmt.Println("END one path")

fmt.Println("START multi path")
g.JSONPathEqs(assert, cluster, validateJSONPaths)
fmt.Println("END multi path")

fmt.Println("START all paths")
// Test validating all paths in golden image
jsonPaths := utils.GetTerminalJSONPaths(g.GetJSON())

// List of paths exempt from validation
exemptJSONPathPrefixes := []string{
"nodePools", // nodePools are unordered
"monitoringConfig.componentConfig.enableComponents",
}

// Remove exempt paths by prefix
jsonPaths = slices.DeleteFunc(jsonPaths, func(s string) bool {
for _, path := range exemptJSONPathPrefixes {
if strings.HasPrefix(s, path) {
// prefix match
return true
}
}
// no prefix match
return false
})

jsonPaths = append(jsonPaths, "monitoringConfig.componentConfig.enableComponents")
// TODO replace with g.JSONPathEqs(assert, cluster, jsonPaths) in bpt v0.17.2

// JSONPathEqs asserts that json content in jsonPaths for got and goldenfile are the same
syncGroup := new(errgroup.Group)
syncGroup.SetLimit(24)
t.Logf("Checking %d JSON paths with max %d goroutines", len(jsonPaths), 20)
for _, jsonPath := range jsonPaths {
jsonPath := jsonPath
syncGroup.Go(func() error {
g.JSONEq(assert, cluster, jsonPath)
return nil
})
}
if err := syncGroup.Wait(); err != nil {
t.Fatal(err)
}
fmt.Println("END all paths")

// Pool-01
assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists")
assert.Equal("e2-medium", cluster.Get("nodePools.#(name==\"pool-01\").config.machineType").String(), "is the expected machine type")
Expand Down Expand Up @@ -129,7 +217,7 @@ func TestNodePool(t *testing.T) {
k8sOpts := k8s.KubectlOptions{}
clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json")
assert.NoError(err)
clusterNodes := testutils.ParseKubectlJSONResult(t, clusterNodesOp)
clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp)
assert.JSONEq(`[
{
"effect": "PreferNoSchedule",
Expand All @@ -148,6 +236,11 @@ func TestNodePool(t *testing.T) {
"effect": "PreferNoSchedule",
"key": "all-pools-example",
"value": "true"
},
{
"effect": "NoSchedule",
"key": "nvidia.com/gpu",
"value": "present"
}
]`,
clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-02\").spec.taints").String(), "has the expected all-pools-example taint")
Expand All @@ -156,6 +249,11 @@ func TestNodePool(t *testing.T) {
"effect": "PreferNoSchedule",
"key": "all-pools-example",
"value": "true"
},
{
"effect": "NoSchedule",
"key": "sandbox.gke.io/runtime",
"value": "gvisor"
}
]`,
clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-03\").spec.taints").String(), "has the expected all-pools-example taint")
Expand Down
Loading

0 comments on commit 2df4b54

Please sign in to comment.