diff --git a/test/manifests/run-test.yaml b/test/manifests/run-test.yaml index 67c410e09d4b..ede0f99641a2 100644 --- a/test/manifests/run-test.yaml +++ b/test/manifests/run-test.yaml @@ -39,4 +39,4 @@ spec: workingDir: $(workspaces.ws.path) script: | aws eks update-kubeconfig --name $(params.cluster-name) - KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" make e2etests + KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" GIT_REF="$(params.git-ref)" make e2etests diff --git a/test/pkg/environment/aws/metrics.go b/test/pkg/environment/aws/metrics.go index 46ae6753eee7..2133461e96e5 100644 --- a/test/pkg/environment/aws/metrics.go +++ b/test/pkg/environment/aws/metrics.go @@ -24,6 +24,8 @@ import ( . "github.com/onsi/gomega" //nolint:revive,stylecheck "github.com/samber/lo" + "github.com/aws/karpenter/test/pkg/environment/common" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" ) @@ -50,6 +52,7 @@ const ( TestSubEventTypeDimension = "subEventType" TestGroupDimension = "group" TestNameDimension = "name" + GitRefDimension = "gitRef" ) // MeasureDurationFor observes the duration between the beginning of the function f() and the end of the function f() @@ -66,7 +69,8 @@ func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group, func (env *Environment) ExpectEventDurationMetric(d time.Duration, labels map[string]string) { GinkgoHelper() - env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), labels) + gitRef := lo.Ternary(env.Context.Value(common.GitRefContextKey) != nil, env.Value(common.GitRefContextKey).(string), "n/a") + env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), lo.Assign(labels, map[string]string{GitRefDimension: gitRef})) } func (env *Environment) ExpectMetric(name string, unit string, value float64, labels map[string]string) { diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index 69f710c01f9e..8b10af171f68 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -38,6 +38,12 @@ import ( "github.com/aws/karpenter/pkg/utils/project" ) +type ContextKey string + +const ( + GitRefContextKey = ContextKey("gitRef") +) + type Environment struct { context.Context @@ -56,7 +62,7 @@ func NewEnvironment(t *testing.T) *Environment { lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter")) kubernetesInterface := kubernetes.NewForConfigOrDie(config) - ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...) + ctx = context.WithValue(injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...), GitRefContextKey, os.Getenv("GIT_REF")) gomega.SetDefaultEventuallyTimeout(5 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index 54c327769752..5239974d68e4 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -318,6 +318,46 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { }, SpecTimeout(time.Hour)) }) Context("Consolidation", func() { + It("should delete all empty nodes with consolidation", func(_ context.Context) { + replicasPerNode := 20 + maxPodDensity := replicasPerNode + dsCount + expectedNodeCount := 200 + replicas := replicasPerNode * expectedNodeCount + + deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) + provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ + MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), + } + + By("waiting for the deployment to deploy all of its pods") + env.ExpectCreated(deployment) + env.EventuallyExpectPendingPodCount(selector, replicas) + + env.MeasureDurationFor(func() { + By("kicking off provisioning by applying the provisioner and nodeTemplate") + env.ExpectCreated(provisioner, nodeTemplate) + + env.EventuallyExpectCreatedMachineCount("==", expectedNodeCount) + env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) + env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) + env.EventuallyExpectHealthyPodCount(selector, replicas) + }, aws.ProvisioningEventType, consolidationTestGroup, "empty/delete") + + env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time + + // Delete deployment to make nodes empty + env.ExpectDeleted(deployment) + env.EventuallyExpectHealthyPodCount(selector, 0) + + env.MeasureDurationFor(func() { + By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner") + provisioner.Spec.Consolidation = &v1alpha5.Consolidation{Enabled: lo.ToPtr(true)} + env.ExpectUpdated(provisioner) + + env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount) + env.EventuallyExpectNodeCount("==", 0) + }, aws.DeprovisioningEventType, consolidationTestGroup, "empty/delete") + }, SpecTimeout(time.Minute*30)) It("should consolidate nodes to get a higher utilization (multi-consolidation delete)", func(_ context.Context) { replicasPerNode := 20 maxPodDensity := replicasPerNode + dsCount @@ -348,6 +388,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { replicas = int(float64(replicas) * 0.2) deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) env.ExpectUpdated(deployment) + env.EventuallyExpectHealthyPodCount(selector, replicas) env.MeasureDurationFor(func() { By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner") @@ -438,8 +479,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() { env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time By("waiting for all deployment pods to be deleted") - // Fully scale down all pods to make nodes empty - deployment.Spec.Replicas = lo.ToPtr[int32](0) + // Delete deployment to make nodes empty env.ExpectDeleted(deployment) env.EventuallyExpectHealthyPodCount(selector, 0)