From 0eaef9d10cf1bc180a3db5a7c87cc226fcdb3a2a Mon Sep 17 00:00:00 2001 From: njtran Date: Thu, 25 May 2023 15:37:33 -0700 Subject: [PATCH 1/4] fix tests and add commit ref in metrics --- Makefile | 3 +- test/manifests/run-test.yaml | 2 +- test/pkg/environment/aws/metrics.go | 4 +- test/pkg/environment/common/environment.go | 4 +- test/suites/scale/deprovisioning_test.go | 76 +++++++++++++++++----- test/suites/scale/provisioning_test.go | 4 +- test/suites/scale/suite_test.go | 7 ++ 7 files changed, 76 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 18d43877f8a5..8ec7be3a033d 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting- # Common Directories MOD_DIRS = $(shell find . -name go.mod -type f | xargs dirname) KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' github.com/aws/karpenter-core) +GIT_REF = $(shell git rev-parse HEAD) # TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" or "make test" against TEST_SUITE ?= "..." @@ -79,7 +80,7 @@ battletest: ## Run randomized, racing, code-covered tests -tags random_test_delay e2etests: ## Run the e2e suite against your local cluster - cd test && CLUSTER_NAME=${CLUSTER_NAME} go test \ + cd test && GIT_REF=${GIT_REF} CLUSTER_NAME=${CLUSTER_NAME} go test \ -p 1 \ -count 1 \ -timeout 180m \ diff --git a/test/manifests/run-test.yaml b/test/manifests/run-test.yaml index 67c410e09d4b..ede0f99641a2 100644 --- a/test/manifests/run-test.yaml +++ b/test/manifests/run-test.yaml @@ -39,4 +39,4 @@ spec: workingDir: $(workspaces.ws.path) script: | aws eks update-kubeconfig --name $(params.cluster-name) - KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" make e2etests + KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" GIT_REF="$(params.git-ref)" make e2etests diff --git a/test/pkg/environment/aws/metrics.go b/test/pkg/environment/aws/metrics.go index 46ae6753eee7..ee9a02344fe4 100644 --- a/test/pkg/environment/aws/metrics.go +++ b/test/pkg/environment/aws/metrics.go @@ -50,10 +50,11 @@ const ( TestSubEventTypeDimension = "subEventType" TestGroupDimension = "group" TestNameDimension = "name" + CommitHashDimension = "commitHash" ) // MeasureDurationFor observes the duration between the beginning of the function f() and the end of the function f() -func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, name string, additionalLabels ...map[string]string) { +func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, name, commit string, additionalLabels ...map[string]string) { GinkgoHelper() start := time.Now() f() @@ -61,6 +62,7 @@ func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, TestEventTypeDimension: string(eventType), TestGroupDimension: group, TestNameDimension: name, + CommitHashDimension: commit, })...)) } diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index 69f710c01f9e..d66d339366dd 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -38,6 +38,8 @@ import ( "github.com/aws/karpenter/pkg/utils/project" ) +const GitRefKey = "gitRef" + type Environment struct { context.Context @@ -56,7 +58,7 @@ func NewEnvironment(t *testing.T) *Environment { lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter")) kubernetesInterface := kubernetes.NewForConfigOrDie(config) - ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...) + ctx = context.WithValue(injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...), GitRefKey, os.Getenv("GIT_REF")) gomega.SetDefaultEventuallyTimeout(5 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index 54c327769752..7f4f31f43ee2 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -233,7 +233,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { }(d) } wg.Wait() - }, aws.ProvisioningEventType, multipleDeprovisionersTestGroup, defaultTestName) + }, aws.ProvisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -310,14 +310,54 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { } env.EventuallyExpectNodeCountWithSelector("==", assertions.nodeCount, selector) env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deploymentMap[d].Spec.Selector.MatchLabels), int(lo.FromPtr(deploymentMap[d].Spec.Replicas))) - }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, map[string]string{aws.TestSubEventTypeDimension: d}) + }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef, map[string]string{aws.TestSubEventTypeDimension: d}) }(k, v) } wg.Wait() - }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName) + }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef) }, SpecTimeout(time.Hour)) }) Context("Consolidation", func() { + It("should delete all empty nodes with consolidation", func (_ context.Context) { + replicasPerNode := 20 + maxPodDensity := replicasPerNode + dsCount + expectedNodeCount := 200 + replicas := replicasPerNode * expectedNodeCount + + deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) + provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ + MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), + } + + By("waiting for the deployment to deploy all of its pods") + env.ExpectCreated(deployment) + env.EventuallyExpectPendingPodCount(selector, replicas) + + env.MeasureDurationFor(func() { + By("kicking off provisioning by applying the provisioner and nodeTemplate") + env.ExpectCreated(provisioner, nodeTemplate) + + env.EventuallyExpectCreatedMachineCount("==", expectedNodeCount) + env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) + env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) + env.EventuallyExpectHealthyPodCount(selector, replicas) + }, aws.ProvisioningEventType, consolidationTestGroup, "empty/delete", gitRef) + + env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time + + // Delete deployment to make nodes empty + env.ExpectDeleted(deployment) + env.EventuallyExpectHealthyPodCount(selector, 0) + + env.MeasureDurationFor(func() { + By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner") + provisioner.Spec.Consolidation = &v1alpha5.Consolidation{Enabled: lo.ToPtr(true)} + env.ExpectUpdated(provisioner) + + env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) + env.EventuallyExpectNodeCount("==", 0) + }, aws.DeprovisioningEventType, consolidationTestGroup, "empty/delete", gitRef) + }, SpecTimeout(time.Minute*30)) It("should consolidate nodes to get a higher utilization (multi-consolidation delete)", func(_ context.Context) { replicasPerNode := 20 maxPodDensity := replicasPerNode + dsCount @@ -341,13 +381,14 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, consolidationTestGroup, "delete") + }, aws.ProvisioningEventType, consolidationTestGroup, "delete", gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time replicas = int(float64(replicas) * 0.2) deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) env.ExpectUpdated(deployment) + env.EventuallyExpectHealthyPodCount(selector, replicas) env.MeasureDurationFor(func() { By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner") @@ -357,7 +398,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", int(float64(expectedNodeCount)*0.8)) env.EventuallyExpectNodeCount("==", int(float64(expectedNodeCount)*0.2)) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, consolidationTestGroup, "delete") + }, aws.DeprovisioningEventType, consolidationTestGroup, "delete", gitRef) }, SpecTimeout(time.Minute*30)) It("should consolidate nodes to get a higher utilization (single consolidation replace)", func(_ context.Context) { replicasPerNode := 1 @@ -389,7 +430,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, consolidationTestGroup, "replace") + }, aws.ProvisioningEventType, consolidationTestGroup, "replace", gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -406,11 +447,11 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) // every node should delete due to replacement env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, consolidationTestGroup, "replace") + }, aws.DeprovisioningEventType, consolidationTestGroup, "replace", gitRef) }, SpecTimeout(time.Hour)) }) Context("Emptiness", func() { - It("should deprovision all nodes when empty", func(_ context.Context) { + FIt("should deprovision all nodes when empty", func(_ context.Context) { replicasPerNode := 20 maxPodDensity := replicasPerNode + dsCount expectedNodeCount := 200 @@ -433,13 +474,12 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, emptinessTestGroup, defaultTestName) + }, aws.ProvisioningEventType, emptinessTestGroup, defaultTestName, gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time By("waiting for all deployment pods to be deleted") - // Fully scale down all pods to make nodes empty - deployment.Spec.Replicas = lo.ToPtr[int32](0) + // Delete deployment to make nodes empty env.ExpectDeleted(deployment) env.EventuallyExpectHealthyPodCount(selector, 0) @@ -450,7 +490,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", 0) - }, aws.DeprovisioningEventType, emptinessTestGroup, defaultTestName) + }, aws.DeprovisioningEventType, emptinessTestGroup, defaultTestName, gitRef) }, SpecTimeout(time.Minute*30)) }) Context("Expiration", func() { @@ -477,7 +517,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, expirationTestGroup, defaultTestName) + }, aws.ProvisioningEventType, expirationTestGroup, defaultTestName, gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -497,7 +537,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, expirationTestGroup, defaultTestName) + }, aws.DeprovisioningEventType, expirationTestGroup, defaultTestName, gitRef) }, SpecTimeout(time.Hour)) }) Context("Drift", func() { @@ -525,7 +565,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, driftTestGroup, defaultTestName) + }, aws.ProvisioningEventType, driftTestGroup, defaultTestName, gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -537,7 +577,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, driftTestGroup, defaultTestName) + }, aws.DeprovisioningEventType, driftTestGroup, defaultTestName, gitRef) }, SpecTimeout(time.Hour)) }) Context("Interruption", func() { @@ -566,7 +606,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) nodes = env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, interruptionTestGroup, defaultTestName) + }, aws.ProvisioningEventType, interruptionTestGroup, defaultTestName, gitRef) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -584,7 +624,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, interruptionTestGroup, defaultTestName) + }, aws.DeprovisioningEventType, interruptionTestGroup, defaultTestName, gitRef) }, SpecTimeout(time.Minute*30)) }) }) diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index 5e8bf31e5288..b37cd15bc989 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -113,7 +113,7 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, testGroup, "pod-dense") + }, aws.ProvisioningEventType, testGroup, "pod-dense", gitRef) }, SpecTimeout(time.Minute*30)) It("should scale successfully on a pod-dense scale-up", func(_ context.Context) { replicasPerNode := 110 @@ -144,6 +144,6 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, testGroup, "node-dense") + }, aws.ProvisioningEventType, testGroup, "node-dense", gitRef) }, SpecTimeout(time.Minute*30)) }) diff --git a/test/suites/scale/suite_test.go b/test/suites/scale/suite_test.go index abae10114f31..824c09b8ce00 100644 --- a/test/suites/scale/suite_test.go +++ b/test/suites/scale/suite_test.go @@ -22,14 +22,21 @@ import ( . "github.com/onsi/gomega" "github.com/aws/karpenter/test/pkg/environment/aws" + "github.com/aws/karpenter/test/pkg/environment/common" ) var env *aws.Environment +var gitRef string func TestScale(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { env = aws.NewEnvironment(t) + if env.Context.Value(common.GitRefKey) != nil { + gitRef = env.Value(common.GitRefKey).(string) + } else { + gitRef = "n/a" + } SetDefaultEventuallyTimeout(time.Hour) env.ExpectPrefixDelegationEnabled() From 77826ae326edc4f2d0caaa60e8098ba635d7aba0 Mon Sep 17 00:00:00 2001 From: njtran Date: Thu, 25 May 2023 15:39:21 -0700 Subject: [PATCH 2/4] fix --- test/suites/scale/deprovisioning_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index 7f4f31f43ee2..64faa1bbbf49 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -451,7 +451,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { }, SpecTimeout(time.Hour)) }) Context("Emptiness", func() { - FIt("should deprovision all nodes when empty", func(_ context.Context) { + It("should deprovision all nodes when empty", func(_ context.Context) { replicasPerNode := 20 maxPodDensity := replicasPerNode + dsCount expectedNodeCount := 200 From 54b99d7cbd0f5aa40f972f69d58fd3c5a4eb939e Mon Sep 17 00:00:00 2001 From: njtran Date: Fri, 26 May 2023 09:25:49 -0700 Subject: [PATCH 3/4] comments --- Makefile | 3 +- test/pkg/environment/aws/metrics.go | 10 +++--- test/pkg/environment/common/environment.go | 8 +++-- test/suites/scale/deprovisioning_test.go | 36 +++++++++++----------- test/suites/scale/provisioning_test.go | 4 +-- test/suites/scale/suite_test.go | 4 +-- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 8ec7be3a033d..18d43877f8a5 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,6 @@ GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting- # Common Directories MOD_DIRS = $(shell find . -name go.mod -type f | xargs dirname) KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' github.com/aws/karpenter-core) -GIT_REF = $(shell git rev-parse HEAD) # TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" or "make test" against TEST_SUITE ?= "..." @@ -80,7 +79,7 @@ battletest: ## Run randomized, racing, code-covered tests -tags random_test_delay e2etests: ## Run the e2e suite against your local cluster - cd test && GIT_REF=${GIT_REF} CLUSTER_NAME=${CLUSTER_NAME} go test \ + cd test && CLUSTER_NAME=${CLUSTER_NAME} go test \ -p 1 \ -count 1 \ -timeout 180m \ diff --git a/test/pkg/environment/aws/metrics.go b/test/pkg/environment/aws/metrics.go index ee9a02344fe4..2133461e96e5 100644 --- a/test/pkg/environment/aws/metrics.go +++ b/test/pkg/environment/aws/metrics.go @@ -24,6 +24,8 @@ import ( . "github.com/onsi/gomega" //nolint:revive,stylecheck "github.com/samber/lo" + "github.com/aws/karpenter/test/pkg/environment/common" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" ) @@ -50,11 +52,11 @@ const ( TestSubEventTypeDimension = "subEventType" TestGroupDimension = "group" TestNameDimension = "name" - CommitHashDimension = "commitHash" + GitRefDimension = "gitRef" ) // MeasureDurationFor observes the duration between the beginning of the function f() and the end of the function f() -func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, name, commit string, additionalLabels ...map[string]string) { +func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, name string, additionalLabels ...map[string]string) { GinkgoHelper() start := time.Now() f() @@ -62,13 +64,13 @@ func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, TestEventTypeDimension: string(eventType), TestGroupDimension: group, TestNameDimension: name, - CommitHashDimension: commit, })...)) } func (env *Environment) ExpectEventDurationMetric(d time.Duration, labels map[string]string) { GinkgoHelper() - env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), labels) + gitRef := lo.Ternary(env.Context.Value(common.GitRefContextKey) != nil, env.Value(common.GitRefContextKey).(string), "n/a") + env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), lo.Assign(labels, map[string]string{GitRefDimension: gitRef})) } func (env *Environment) ExpectMetric(name string, unit string, value float64, labels map[string]string) { diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index d66d339366dd..8b10af171f68 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -38,7 +38,11 @@ import ( "github.com/aws/karpenter/pkg/utils/project" ) -const GitRefKey = "gitRef" +type ContextKey string + +const ( + GitRefContextKey = ContextKey("gitRef") +) type Environment struct { context.Context @@ -58,7 +62,7 @@ func NewEnvironment(t *testing.T) *Environment { lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter")) kubernetesInterface := kubernetes.NewForConfigOrDie(config) - ctx = context.WithValue(injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...), GitRefKey, os.Getenv("GIT_REF")) + ctx = context.WithValue(injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...), GitRefContextKey, os.Getenv("GIT_REF")) gomega.SetDefaultEventuallyTimeout(5 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index 64faa1bbbf49..5239974d68e4 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -233,7 +233,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { }(d) } wg.Wait() - }, aws.ProvisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef) + }, aws.ProvisioningEventType, multipleDeprovisionersTestGroup, defaultTestName) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -310,15 +310,15 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { } env.EventuallyExpectNodeCountWithSelector("==", assertions.nodeCount, selector) env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deploymentMap[d].Spec.Selector.MatchLabels), int(lo.FromPtr(deploymentMap[d].Spec.Replicas))) - }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef, map[string]string{aws.TestSubEventTypeDimension: d}) + }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, map[string]string{aws.TestSubEventTypeDimension: d}) }(k, v) } wg.Wait() - }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName, gitRef) + }, aws.DeprovisioningEventType, multipleDeprovisionersTestGroup, defaultTestName) }, SpecTimeout(time.Hour)) }) Context("Consolidation", func() { - It("should delete all empty nodes with consolidation", func (_ context.Context) { + It("should delete all empty nodes with consolidation", func(_ context.Context) { replicasPerNode := 20 maxPodDensity := replicasPerNode + dsCount expectedNodeCount := 200 @@ -341,7 +341,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, consolidationTestGroup, "empty/delete", gitRef) + }, aws.ProvisioningEventType, consolidationTestGroup, "empty/delete") env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -356,7 +356,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", 0) - }, aws.DeprovisioningEventType, consolidationTestGroup, "empty/delete", gitRef) + }, aws.DeprovisioningEventType, consolidationTestGroup, "empty/delete") }, SpecTimeout(time.Minute*30)) It("should consolidate nodes to get a higher utilization (multi-consolidation delete)", func(_ context.Context) { replicasPerNode := 20 @@ -381,7 +381,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, consolidationTestGroup, "delete", gitRef) + }, aws.ProvisioningEventType, consolidationTestGroup, "delete") env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -398,7 +398,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", int(float64(expectedNodeCount)*0.8)) env.EventuallyExpectNodeCount("==", int(float64(expectedNodeCount)*0.2)) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, consolidationTestGroup, "delete", gitRef) + }, aws.DeprovisioningEventType, consolidationTestGroup, "delete") }, SpecTimeout(time.Minute*30)) It("should consolidate nodes to get a higher utilization (single consolidation replace)", func(_ context.Context) { replicasPerNode := 1 @@ -430,7 +430,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, consolidationTestGroup, "replace", gitRef) + }, aws.ProvisioningEventType, consolidationTestGroup, "replace") env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -447,7 +447,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) // every node should delete due to replacement env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, consolidationTestGroup, "replace", gitRef) + }, aws.DeprovisioningEventType, consolidationTestGroup, "replace") }, SpecTimeout(time.Hour)) }) Context("Emptiness", func() { @@ -474,7 +474,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, emptinessTestGroup, defaultTestName, gitRef) + }, aws.ProvisioningEventType, emptinessTestGroup, defaultTestName) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -490,7 +490,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", 0) - }, aws.DeprovisioningEventType, emptinessTestGroup, defaultTestName, gitRef) + }, aws.DeprovisioningEventType, emptinessTestGroup, defaultTestName) }, SpecTimeout(time.Minute*30)) }) Context("Expiration", func() { @@ -517,7 +517,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, expirationTestGroup, defaultTestName, gitRef) + }, aws.ProvisioningEventType, expirationTestGroup, defaultTestName) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -537,7 +537,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, expirationTestGroup, defaultTestName, gitRef) + }, aws.DeprovisioningEventType, expirationTestGroup, defaultTestName) }, SpecTimeout(time.Hour)) }) Context("Drift", func() { @@ -565,7 +565,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, driftTestGroup, defaultTestName, gitRef) + }, aws.ProvisioningEventType, driftTestGroup, defaultTestName) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -577,7 +577,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, driftTestGroup, defaultTestName, gitRef) + }, aws.DeprovisioningEventType, driftTestGroup, defaultTestName) }, SpecTimeout(time.Hour)) }) Context("Interruption", func() { @@ -606,7 +606,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) nodes = env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, interruptionTestGroup, defaultTestName, gitRef) + }, aws.ProvisioningEventType, interruptionTestGroup, defaultTestName) env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time @@ -624,7 +624,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) env.EventuallyExpectNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.DeprovisioningEventType, interruptionTestGroup, defaultTestName, gitRef) + }, aws.DeprovisioningEventType, interruptionTestGroup, defaultTestName) }, SpecTimeout(time.Minute*30)) }) }) diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index b37cd15bc989..5e8bf31e5288 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -113,7 +113,7 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, testGroup, "pod-dense", gitRef) + }, aws.ProvisioningEventType, testGroup, "pod-dense") }, SpecTimeout(time.Minute*30)) It("should scale successfully on a pod-dense scale-up", func(_ context.Context) { replicasPerNode := 110 @@ -144,6 +144,6 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), func() { env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) env.EventuallyExpectHealthyPodCount(selector, replicas) - }, aws.ProvisioningEventType, testGroup, "node-dense", gitRef) + }, aws.ProvisioningEventType, testGroup, "node-dense") }, SpecTimeout(time.Minute*30)) }) diff --git a/test/suites/scale/suite_test.go b/test/suites/scale/suite_test.go index 824c09b8ce00..0428c293d5e0 100644 --- a/test/suites/scale/suite_test.go +++ b/test/suites/scale/suite_test.go @@ -32,8 +32,8 @@ func TestScale(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { env = aws.NewEnvironment(t) - if env.Context.Value(common.GitRefKey) != nil { - gitRef = env.Value(common.GitRefKey).(string) + if env.Context.Value(common.GitRefContextKey) != nil { + gitRef = env.Value(common.GitRefContextKey).(string) } else { gitRef = "n/a" } From b45ddc74777397404d7a8b4069f7e817a46c3eeb Mon Sep 17 00:00:00 2001 From: njtran Date: Fri, 26 May 2023 09:30:34 -0700 Subject: [PATCH 4/4] save issuesg --- test/suites/scale/suite_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/suites/scale/suite_test.go b/test/suites/scale/suite_test.go index 0428c293d5e0..abae10114f31 100644 --- a/test/suites/scale/suite_test.go +++ b/test/suites/scale/suite_test.go @@ -22,21 +22,14 @@ import ( . "github.com/onsi/gomega" "github.com/aws/karpenter/test/pkg/environment/aws" - "github.com/aws/karpenter/test/pkg/environment/common" ) var env *aws.Environment -var gitRef string func TestScale(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { env = aws.NewEnvironment(t) - if env.Context.Value(common.GitRefContextKey) != nil { - gitRef = env.Value(common.GitRefContextKey).(string) - } else { - gitRef = "n/a" - } SetDefaultEventuallyTimeout(time.Hour) env.ExpectPrefixDelegationEnabled()