From 93897d8d1b978e7c3c4070e3bcc8b0693e193495 Mon Sep 17 00:00:00 2001 From: Yaroslava Serdiuk Date: Tue, 9 Jul 2024 09:43:31 +0000 Subject: [PATCH] Delete old ProvReqs --- .../processors/provreq/processor.go | 27 ++++++++- .../processors/provreq/processor_test.go | 58 ++++++++++++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/cluster-autoscaler/processors/provreq/processor.go b/cluster-autoscaler/processors/provreq/processor.go index a08af27f8a0..810112e94eb 100644 --- a/cluster-autoscaler/processors/provreq/processor.go +++ b/cluster-autoscaler/processors/provreq/processor.go @@ -23,7 +23,7 @@ import ( apiv1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" + v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest" "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/conditions" @@ -33,13 +33,15 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling" + "k8s.io/autoscaler/cluster-autoscaler/utils/klogx" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/framework" ) const ( - defaultReservationTime = 10 * time.Minute - defaultExpirationTime = 7 * 24 * time.Hour // 7 days + defaultReservationTime = 10 * time.Minute + defaultExpirationTime = 7 * 24 * time.Hour // 7 days + defaultTerminalProvReqTTL = 7 * 24 * time.Hour // 7 days // defaultMaxUpdated is a limit for ProvisioningRequest to update conditions in one ClusterAutoscaler loop. defaultMaxUpdated = 20 ) @@ -118,6 +120,7 @@ func (p *provReqProcessor) refresh(provReqs []*provreqwrapper.ProvisioningReques continue } } + p.DeleteOldProvReqs(provReqs) } // CleanUp cleans up internal state @@ -167,3 +170,21 @@ func (p *provReqProcessor) bookCapacity(ctx *context.AutoscalingContext) error { } return nil } + +// DeleteOldProvReqs delete ProvReq that have terminal state (Provisioned/Failed == True) more than a week. +func (p *provReqProcessor) DeleteOldProvReqs(provReqs []*provreqwrapper.ProvisioningRequest) { + provReqQuota := klogx.NewLoggingQuota(30) + for _, provReq := range provReqs { + conditions := provReq.Status.Conditions + provisioned := apimeta.FindStatusCondition(conditions, v1.Provisioned) + failed := apimeta.FindStatusCondition(conditions, v1.Failed) + if provisioned != nil && provisioned.LastTransitionTime.Add(defaultTerminalProvReqTTL).Before(p.now()) || + failed != nil && failed.LastTransitionTime.Add(defaultTerminalProvReqTTL).Before(p.now()) { + klogx.V(4).UpTo(provReqQuota).Infof("Delete old ProvisioningRequest %s/%s", provReq.Namespace, provReq.Name) + err := p.client.DeleteProvisioningRequest(provReq.ProvisioningRequest) + if err != nil { + klog.Warningf("Couldn't delete old %s/%s Provisioning Request, err: %v", provReq.Namespace, provReq.Name, err) + } + } + } +} diff --git a/cluster-autoscaler/processors/provreq/processor_test.go b/cluster-autoscaler/processors/provreq/processor_test.go index e2eb8cae9e9..3265a64ea0f 100644 --- a/cluster-autoscaler/processors/provreq/processor_test.go +++ b/cluster-autoscaler/processors/provreq/processor_test.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/scheduler/framework" - "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" + v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" "k8s.io/autoscaler/cluster-autoscaler/config" . "k8s.io/autoscaler/cluster-autoscaler/core/test" "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/conditions" @@ -150,11 +150,14 @@ func TestRefresh(t *testing.T) { pr.Status.Conditions = test.conditions pr.CreationTimestamp = metav1.NewTime(test.creationTime) pr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity + additionalPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "additional") additionalPr.CreationTimestamp = metav1.NewTime(weekAgo) additionalPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity + processor := provReqProcessor{func() time.Time { return now }, 1, provreqclient.NewFakeProvisioningRequestClient(nil, t, pr, additionalPr), nil} processor.refresh([]*provreqwrapper.ProvisioningRequest{pr, additionalPr}) + assert.ElementsMatch(t, test.wantConditions, pr.Status.Conditions) if len(test.conditions) == len(test.wantConditions) { assert.ElementsMatch(t, []metav1.Condition{ @@ -172,6 +175,59 @@ func TestRefresh(t *testing.T) { } } +func TestDeleteOldProvReqs(t *testing.T) { + now := time.Now() + tenDaysAgo := now.Add(-1 * 10 * 24 * time.Hour) + pr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "name-1") + additionalPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "additional") + + oldFailedPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "failed") + oldExpiredPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "expired") + oldFailedPr.CreationTimestamp = metav1.NewTime(tenDaysAgo) + oldExpiredPr.CreationTimestamp = metav1.NewTime(tenDaysAgo) + oldFailedPr.Status.Conditions = []metav1.Condition{ + { + Type: v1.Failed, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(tenDaysAgo), + Reason: "Failed", + Message: "Failed", + }, + } + oldFailedPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity + oldExpiredPr.Status.Conditions = []metav1.Condition{ + { + Type: v1.Provisioned, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(tenDaysAgo), + Reason: "Provisioned", + Message: "", + }, + { + Type: v1.BookingExpired, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(tenDaysAgo), + Reason: "Capacity is expired", + Message: "", + }, + } + oldExpiredPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity + + client := provreqclient.NewFakeProvisioningRequestClient(nil, t, pr, additionalPr, oldFailedPr, oldExpiredPr) + + processor := provReqProcessor{func() time.Time { return now }, 1, client, nil} + processor.refresh([]*provreqwrapper.ProvisioningRequest{pr, additionalPr, oldFailedPr, oldExpiredPr}) + + _, err := client.ProvisioningRequestNoCache(oldFailedPr.Namespace, oldFailedPr.Name) + assert.Error(t, err) + _, err = client.ProvisioningRequestNoCache(oldExpiredPr.Namespace, oldExpiredPr.Name) + assert.Error(t, err) + _, err = client.ProvisioningRequestNoCache(pr.Namespace, pr.Name) + assert.NoError(t, err) + _, err = client.ProvisioningRequestNoCache(additionalPr.Namespace, additionalPr.Name) + assert.NoError(t, err) +} + type fakeInjector struct { pods []*apiv1.Pod }