Skip to content

Commit

Permalink
switch to glcoud
Browse files Browse the repository at this point in the history
  • Loading branch information
apeabody committed Dec 19, 2024
1 parent cbfc3ab commit 039e64d
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 195 deletions.
155 changes: 2 additions & 153 deletions build/int.cloudbuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,172 +29,21 @@ steps:
- prepare
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage init --verbose']
- id: converge node-pool-local
- id: apply node-pool-local
waitFor:
- init node-pool-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage apply --verbose']
- id: verify node-pool-local
waitFor:
- converge node-pool-local
- apply node-pool-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage verify --verbose']
- id: destroy node-pool-local
waitFor:
- verify node-pool-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage destroy --verbose']
- id: apply sandbox-enabled-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage apply --verbose']
- id: verify sandbox-enabled-local
waitFor:
- apply sandbox-enabled-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage verify --verbose']
- id: destroy sandbox-enabled-local
waitFor:
- verify sandbox-enabled-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSandboxEnabled --stage destroy --verbose']
- id: apply workload-identity-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage apply --verbose']
- id: verify workload-identity-local
waitFor:
- apply workload-identity-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage verify --verbose']
- id: destroy workload-identity-local
waitFor:
- verify workload-identity-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage teardown --verbose']
- id: apply safer-cluster-iap-bastion-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage apply --verbose']
- id: verify safer-cluster-iap-bastion-local
waitFor:
- apply safer-cluster-iap-bastion-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage verify --verbose']
- id: destroy safer-cluster-iap-bastion-local
waitFor:
- verify safer-cluster-iap-bastion-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage teardown --verbose']
- id: apply simple-zonal-with-asm-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage apply --verbose']
- id: verify simple-zonal-with-asm-local
waitFor:
- apply simple-zonal-with-asm-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage verify --verbose']
- id: destroy simple-zonal-with-asm-local
waitFor:
- verify simple-zonal-with-asm-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleZonalWithASM --stage teardown --verbose']
- id: apply simple-autopilot-private-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage apply --verbose']
- id: verify simple-autopilot-private-local
waitFor:
- apply simple-autopilot-private-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage verify --verbose']
- id: destroy simple-autopilot-private-local
waitFor:
- verify simple-autopilot-private-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage teardown --verbose']
- id: apply autopilot-private-firewalls
waitFor:
- create-all
- destroy simple-autopilot-private-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage apply --verbose']
- id: verify autopilot-private-firewalls
waitFor:
- apply autopilot-private-firewalls
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage verify --verbose']
- id: destroy autopilot-private-firewalls
waitFor:
- verify autopilot-private-firewalls
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage destroy --verbose']
- id: apply simple-autopilot-public-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage apply --verbose']
- id: verify simple-autopilot-public-local
waitFor:
- apply simple-autopilot-public-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage verify --verbose']
- id: destroy simple-autopilot-public-local
waitFor:
- verify simple-autopilot-public-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPublic --stage teardown --verbose']
- id: apply private-zonal-with-networking
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage apply --verbose --test-dir test/integration']
- id: verify private-zonal-with-networking
waitFor:
- apply private-zonal-with-networking
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage verify --verbose --test-dir test/integration']
- id: teardown private-zonal-with-networking
waitFor:
- verify private-zonal-with-networking
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration']
- id: apply simple-autopilot-private-non-default-sa
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage apply --verbose']
- id: verify simple-autopilot-private-non-default-sa
waitFor:
- apply simple-autopilot-private-non-default-sa
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage verify --verbose']
- id: teardown simple-autopilot-private-non-default-sa
waitFor:
- verify simple-autopilot-private-non-default-sa
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose']
- id: apply simple-fleet-app-operator-permissions
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage apply --verbose']
- id: verify simple-fleet-app-operator-permissions
waitFor:
- apply simple-fleet-app-operator-permissions
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage verify --verbose']
- id: teardown simple-fleet-app-operator-permissions
waitFor:
- verify simple-fleet-app-operator-permissions
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose']
tags:
- 'ci'
- 'integration'
Expand Down
4 changes: 2 additions & 2 deletions test/integration/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ require (
github.com/gruntwork-io/terratest v0.48.1
github.com/hashicorp/terraform-json v0.24.0
github.com/stretchr/testify v1.10.0
github.com/tidwall/gjson v1.18.0
golang.org/x/sync v0.10.0
)

require (
Expand Down Expand Up @@ -103,7 +105,6 @@ require (
github.com/pquerna/otp v1.4.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
Expand All @@ -116,7 +117,6 @@ require (
golang.org/x/mod v0.22.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/oauth2 v0.24.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/term v0.26.0 // indirect
golang.org/x/text v0.20.0 // indirect
Expand Down
49 changes: 16 additions & 33 deletions test/integration/node_pool/node_pool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,13 @@ package node_pool

import (
"fmt"
"slices"
"testing"
"time"

"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/stretchr/testify/assert"
"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
)
Expand All @@ -47,18 +44,12 @@ func TestNodePool(t *testing.T) {

// Retrieve Project CAI
projectCAI := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster", "k8s.io/Node"}))
t.Log(projectCAI.Raw)
// Retrieve Cluster from CAI
clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName)

if !projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data").Exists() {
t.Fatalf("Cluster not found: %s", clusterResourceName)
}

cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data")
t.Log(cluster.Raw)
// Retrieve Cluster from CAI
// clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName)
// cluster := projectCAI.Get("#(name=\"" + clusterResourceName + "\").resource.data")
// Equivalent gcloud describe command (classic)
// cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)

// Cluster Assertions (classic)
assert.Contains([]string{"RUNNING", "RECONCILING"}, cluster.Get("status").String(), "Cluster is Running")
Expand Down Expand Up @@ -87,19 +78,8 @@ func TestNodePool(t *testing.T) {
golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")),
golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")),
)
checkPaths := utils.GetTerminalJSONPaths(g.GetJSON())

exemptPaths := []string{"nodePools"}
checkPaths = slices.DeleteFunc(checkPaths, func(s string) bool {
return slices.Contains(exemptPaths, s)
})
g.JSONPathEqs(assert, cluster, checkPaths)

// NodePool Assertions
nodePools := []string{"pool-01", "pool-02", "pool-03", "pool-04", "pool-05"}
for _, nodePool := range nodePools {
g.JSONPathEqs(assert, cluster.Get(fmt.Sprintf("nodePools.#(name==%s).name", nodePool)), utils.GetTerminalJSONPaths(g.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s).name", nodePool))))
}
testutils.TGKEAssertGolden(assert, g, &cluster, []string{"pool-01", "pool-02", "pool-03", "pool-04", "pool-05"}, []string{"monitoringConfig.componentConfig.enableComponents"})

// nodePool-01 Assertions
assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists")
Expand Down Expand Up @@ -161,11 +141,14 @@ func TestNodePool(t *testing.T) {
assert.True(cluster.Get("nodePools.#(name==\"pool-05\").config.advancedMachineFeatures.enableNestedVirtualization").Bool(), "has enable_nested_virtualization enabled")

// K8s Assertions
gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId)
k8sOpts := k8s.KubectlOptions{}
clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json")
assert.NoError(err)
clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp)

// Equivalent kubectl command (classic)
// gcloud.Runf(t, "container clusters get-credentials %s --region %s --project %s", clusterName, location, projectId)
// k8sOpts := k8s.KubectlOptions{}
// clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json")
// assert.NoError(err)
// clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp)

assert.JSONEq(`[
{
"effect": "PreferNoSchedule",
Expand All @@ -178,7 +161,7 @@ func TestNodePool(t *testing.T) {
"value": "true"
}
]`,
clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-01\").spec.taints").String(), "has the expected taints")
projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-01\").resource.data.spec.taints").String(), "has the expected taints")
assert.JSONEq(`[
{
"effect": "PreferNoSchedule",
Expand All @@ -191,7 +174,7 @@ func TestNodePool(t *testing.T) {
"value": "present"
}
]`,
clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-02\").spec.taints").String(), "has the expected all-pools-example taint")
projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-02\").resource.data.spec.taints").String(), "has the expected all-pools-example taint")
assert.JSONEq(`[
{
"effect": "PreferNoSchedule",
Expand All @@ -204,7 +187,7 @@ func TestNodePool(t *testing.T) {
"value": "gvisor"
}
]`,
clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-03\").spec.taints").String(), "has the expected all-pools-example taint")
projectCAI.Get("#(resource.data.metadata.labels.node_pool==\"pool-03\").resource.data.spec.taints").String(), "has the expected all-pools-example taint")
})

bpt.Test()
Expand Down
4 changes: 2 additions & 2 deletions test/integration/node_pool/testdata/TestNodePool.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"strategy": "SURGE"
}
},
"autoscalingProfile": "OPTIMIZE_UTILIZATION",
"autoscalingProfile": "OPTIMIZE_UTILIZATIONa",
"enableNodeAutoprovisioning": true,
"resourceLimits": [
{
Expand Down Expand Up @@ -715,7 +715,7 @@
"variant": "DEFAULT"
}
},
"machineType": "n1-standard-2",
"machineType": "n1-standard-2a",
"metadata": {
"cluster_name": "node-pool-cluster-RANDOM_STRING",
"disable-legacy-endpoints": "false",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,20 +35,20 @@ func TestSaferClusterIapBastion(t *testing.T) {
// bpt.DefaultVerify(assert)
testutils.TGKEVerify(t, bpt, assert) // Verify Resources

test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ")
testCommand, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ")

// pre run ssh command so that ssh-keygen can run
gcloud.RunCmd(t, test_command,
gcloud.RunCmd(t, testCommand,
gcloud.WithCommonArgs([]string{}),
)

cluster_version := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version"))
clusterVersion := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version"))

op := gcloud.Run(t, test_command,
op := gcloud.Run(t, testCommand,
gcloud.WithCommonArgs([]string{}),
)

assert.Equal(cluster_version, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE")
assert.Equal(clusterVersion, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE")
})

bpt.Test()
Expand Down
Loading

0 comments on commit 039e64d

Please sign in to comment.