test: fix consolidation test timing and add commit ref in metrics (#3942

)

Co-authored-by: njtran <[email protected]>

test/manifests/run-test.yaml

@@ -39,4 +39,4 @@ spec:
     workingDir: $(workspaces.ws.path)
     script: |
       aws eks update-kubeconfig --name $(params.cluster-name)
-      KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" make e2etests
+      KUBECONFIG=/root/.kube/config ENABLE_CLOUDWATCH=true TEST_SUITE="$(params.test-filter)" GIT_REF="$(params.git-ref)" make e2etests

test/pkg/environment/aws/metrics.go

@@ -24,6 +24,8 @@ import (
 	. "github.com/onsi/gomega"    //nolint:revive,stylecheck
 	"github.com/samber/lo"
 
+	"github.com/aws/karpenter/test/pkg/environment/common"
+
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
 )
 
@@ -50,6 +52,7 @@ const (
 	TestSubEventTypeDimension   = "subEventType"
 	TestGroupDimension          = "group"
 	TestNameDimension           = "name"
+	GitRefDimension             = "gitRef"
 )
 
 // MeasureDurationFor observes the duration between the beginning of the function f() and the end of the function f()
@@ -66,7 +69,8 @@ func (env *Environment) MeasureDurationFor(f func(), eventType EventType, group,
 
 func (env *Environment) ExpectEventDurationMetric(d time.Duration, labels map[string]string) {
 	GinkgoHelper()
-	env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), labels)
+	gitRef := lo.Ternary(env.Context.Value(common.GitRefContextKey) != nil, env.Value(common.GitRefContextKey).(string), "n/a")
+	env.ExpectMetric("eventDuration", cloudwatch.StandardUnitSeconds, d.Seconds(), lo.Assign(labels, map[string]string{GitRefDimension: gitRef}))
 }
 
 func (env *Environment) ExpectMetric(name string, unit string, value float64, labels map[string]string) {

test/pkg/environment/common/environment.go

@@ -38,6 +38,12 @@ import (
 	"github.com/aws/karpenter/pkg/utils/project"
 )
 
+type ContextKey string
+
+const (
+	GitRefContextKey = ContextKey("gitRef")
+)
+
 type Environment struct {
 	context.Context
 
@@ -56,7 +62,7 @@ func NewEnvironment(t *testing.T) *Environment {
 
 	lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter"))
 	kubernetesInterface := kubernetes.NewForConfigOrDie(config)
-	ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...)
+	ctx = context.WithValue(injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...), GitRefContextKey, os.Getenv("GIT_REF"))
 
 	gomega.SetDefaultEventuallyTimeout(5 * time.Minute)
 	gomega.SetDefaultEventuallyPollingInterval(1 * time.Second)

test/suites/scale/deprovisioning_test.go

@@ -318,6 +318,46 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() {
 		}, SpecTimeout(time.Hour))
 	})
 	Context("Consolidation", func() {
+		It("should delete all empty nodes with consolidation", func(_ context.Context) {
+			replicasPerNode := 20
+			maxPodDensity := replicasPerNode + dsCount
+			expectedNodeCount := 200
+			replicas := replicasPerNode * expectedNodeCount
+
+			deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas))
+			provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{
+				MaxPods: lo.ToPtr[int32](int32(maxPodDensity)),
+			}
+
+			By("waiting for the deployment to deploy all of its pods")
+			env.ExpectCreated(deployment)
+			env.EventuallyExpectPendingPodCount(selector, replicas)
+
+			env.MeasureDurationFor(func() {
+				By("kicking off provisioning by applying the provisioner and nodeTemplate")
+				env.ExpectCreated(provisioner, nodeTemplate)
+
+				env.EventuallyExpectCreatedMachineCount("==", expectedNodeCount)
+				env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount)
+				env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount)
+				env.EventuallyExpectHealthyPodCount(selector, replicas)
+			}, aws.ProvisioningEventType, consolidationTestGroup, "empty/delete")
+
+			env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time
+
+			// Delete deployment to make nodes empty
+			env.ExpectDeleted(deployment)
+			env.EventuallyExpectHealthyPodCount(selector, 0)
+
+			env.MeasureDurationFor(func() {
+				By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner")
+				provisioner.Spec.Consolidation = &v1alpha5.Consolidation{Enabled: lo.ToPtr(true)}
+				env.ExpectUpdated(provisioner)
+
+				env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount)
+				env.EventuallyExpectNodeCount("==", 0)
+			}, aws.DeprovisioningEventType, consolidationTestGroup, "empty/delete")
+		}, SpecTimeout(time.Minute*30))
 		It("should consolidate nodes to get a higher utilization (multi-consolidation delete)", func(_ context.Context) {
 			replicasPerNode := 20
 			maxPodDensity := replicasPerNode + dsCount
@@ -348,6 +388,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() {
 			replicas = int(float64(replicas) * 0.2)
 			deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas))
 			env.ExpectUpdated(deployment)
+			env.EventuallyExpectHealthyPodCount(selector, replicas)
 
 			env.MeasureDurationFor(func() {
 				By("kicking off deprovisioning by setting the consolidation enabled value on the provisioner")
@@ -438,8 +479,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), func() {
 			env.Monitor.Reset() // Reset the monitor so that we now track the nodes starting at this point in time
 
 			By("waiting for all deployment pods to be deleted")
-			// Fully scale down all pods to make nodes empty
-			deployment.Spec.Replicas = lo.ToPtr[int32](0)
+			// Delete deployment to make nodes empty
 			env.ExpectDeleted(deployment)
 			env.EventuallyExpectHealthyPodCount(selector, 0)