Skip to content

Commit

Permalink
Delete old ProvReqs
Browse files Browse the repository at this point in the history
  • Loading branch information
@yaroslava-serdiuk
yaroslava-serdiuk committed Sep 11, 2024
1 parent cecb34c commit 93897d8
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 4 deletions.
27 changes: 24 additions & 3 deletions cluster-autoscaler/processors/provreq/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
apiv1 "k8s.io/api/core/v1"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/conditions"
Expand All @@ -33,13 +33,15 @@ import (
"k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot"
"k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker"
"k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling"
"k8s.io/autoscaler/cluster-autoscaler/utils/klogx"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/scheduler/framework"
)

const (
defaultReservationTime = 10 * time.Minute
defaultExpirationTime = 7 * 24 * time.Hour // 7 days
defaultReservationTime = 10 * time.Minute
defaultExpirationTime = 7 * 24 * time.Hour // 7 days
defaultTerminalProvReqTTL = 7 * 24 * time.Hour // 7 days
// defaultMaxUpdated is a limit for ProvisioningRequest to update conditions in one ClusterAutoscaler loop.
defaultMaxUpdated = 20
)
Expand Down Expand Up @@ -118,6 +120,7 @@ func (p *provReqProcessor) refresh(provReqs []*provreqwrapper.ProvisioningReques
continue
}
}
p.DeleteOldProvReqs(provReqs)
}

// CleanUp cleans up internal state
Expand Down Expand Up @@ -167,3 +170,21 @@ func (p *provReqProcessor) bookCapacity(ctx *context.AutoscalingContext) error {
}
return nil
}

// DeleteOldProvReqs delete ProvReq that have terminal state (Provisioned/Failed == True) more than a week.
func (p *provReqProcessor) DeleteOldProvReqs(provReqs []*provreqwrapper.ProvisioningRequest) {
provReqQuota := klogx.NewLoggingQuota(30)
for _, provReq := range provReqs {
conditions := provReq.Status.Conditions
provisioned := apimeta.FindStatusCondition(conditions, v1.Provisioned)
failed := apimeta.FindStatusCondition(conditions, v1.Failed)
if provisioned != nil && provisioned.LastTransitionTime.Add(defaultTerminalProvReqTTL).Before(p.now()) ||
failed != nil && failed.LastTransitionTime.Add(defaultTerminalProvReqTTL).Before(p.now()) {
klogx.V(4).UpTo(provReqQuota).Infof("Delete old ProvisioningRequest %s/%s", provReq.Namespace, provReq.Name)
err := p.client.DeleteProvisioningRequest(provReq.ProvisioningRequest)
if err != nil {
klog.Warningf("Couldn't delete old %s/%s Provisioning Request, err: %v", provReq.Namespace, provReq.Name, err)
}
}
}
}
58 changes: 57 additions & 1 deletion cluster-autoscaler/processors/provreq/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/scheduler/framework"

"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
"k8s.io/autoscaler/cluster-autoscaler/config"
. "k8s.io/autoscaler/cluster-autoscaler/core/test"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/conditions"
Expand Down Expand Up @@ -150,11 +150,14 @@ func TestRefresh(t *testing.T) {
pr.Status.Conditions = test.conditions
pr.CreationTimestamp = metav1.NewTime(test.creationTime)
pr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity

additionalPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "additional")
additionalPr.CreationTimestamp = metav1.NewTime(weekAgo)
additionalPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity

processor := provReqProcessor{func() time.Time { return now }, 1, provreqclient.NewFakeProvisioningRequestClient(nil, t, pr, additionalPr), nil}
processor.refresh([]*provreqwrapper.ProvisioningRequest{pr, additionalPr})

assert.ElementsMatch(t, test.wantConditions, pr.Status.Conditions)
if len(test.conditions) == len(test.wantConditions) {
assert.ElementsMatch(t, []metav1.Condition{
Expand All @@ -172,6 +175,59 @@ func TestRefresh(t *testing.T) {
}
}

func TestDeleteOldProvReqs(t *testing.T) {
now := time.Now()
tenDaysAgo := now.Add(-1 * 10 * 24 * time.Hour)
pr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "name-1")
additionalPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "additional")

oldFailedPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "failed")
oldExpiredPr := provreqclient.ProvisioningRequestWrapperForTesting("namespace", "expired")
oldFailedPr.CreationTimestamp = metav1.NewTime(tenDaysAgo)
oldExpiredPr.CreationTimestamp = metav1.NewTime(tenDaysAgo)
oldFailedPr.Status.Conditions = []metav1.Condition{
{
Type: v1.Failed,
Status: metav1.ConditionTrue,
LastTransitionTime: metav1.NewTime(tenDaysAgo),
Reason: "Failed",
Message: "Failed",
},
}
oldFailedPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity
oldExpiredPr.Status.Conditions = []metav1.Condition{
{
Type: v1.Provisioned,
Status: metav1.ConditionTrue,
LastTransitionTime: metav1.NewTime(tenDaysAgo),
Reason: "Provisioned",
Message: "",
},
{
Type: v1.BookingExpired,
Status: metav1.ConditionTrue,
LastTransitionTime: metav1.NewTime(tenDaysAgo),
Reason: "Capacity is expired",
Message: "",
},
}
oldExpiredPr.Spec.ProvisioningClassName = v1.ProvisioningClassCheckCapacity

client := provreqclient.NewFakeProvisioningRequestClient(nil, t, pr, additionalPr, oldFailedPr, oldExpiredPr)

processor := provReqProcessor{func() time.Time { return now }, 1, client, nil}
processor.refresh([]*provreqwrapper.ProvisioningRequest{pr, additionalPr, oldFailedPr, oldExpiredPr})

_, err := client.ProvisioningRequestNoCache(oldFailedPr.Namespace, oldFailedPr.Name)
assert.Error(t, err)
_, err = client.ProvisioningRequestNoCache(oldExpiredPr.Namespace, oldExpiredPr.Name)
assert.Error(t, err)
_, err = client.ProvisioningRequestNoCache(pr.Namespace, pr.Name)
assert.NoError(t, err)
_, err = client.ProvisioningRequestNoCache(additionalPr.Namespace, additionalPr.Name)
assert.NoError(t, err)
}

type fakeInjector struct {
pods []*apiv1.Pod
}
Expand Down

0 comments on commit 93897d8

Please sign in to comment.