Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cluster-autoscaler-release-1.30] Update ProvReq annotations to new prefix #7060

Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -200,4 +200,8 @@ const (
// ProvisioningClassBestEffortAtomicScaleUp denotes that CA try to provision the capacity
// in an atomic manner.
ProvisioningClassBestEffortAtomicScaleUp string = "best-effort-atomic-scale-up.autoscaling.x-k8s.io"
// ProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request.
ProvisioningRequestPodAnnotationKey = "autoscaling.x-k8s.io/consume-provisioning-request"
// ProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class
ProvisioningClassPodAnnotationKey = "autoscaling.x-k8s.io/provisioning-class-name"
)
13 changes: 7 additions & 6 deletions cluster-autoscaler/processors/provreq/pods_filter.go
Original file line number Diff line number Diff line change
@@ -22,16 +22,14 @@ import (

apiv1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
provreqpods "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
"k8s.io/autoscaler/cluster-autoscaler/utils/klogx"
)

const (
// ProvisioningRequestPodAnnotationKey is an annotation on pod that indicate that pod was created by ProvisioningRequest.
ProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request"
maxProvReqEvent = 50
)
const maxProvReqEvent = 50

// EventManager is an interface for handling events for provisioning request.
type EventManager interface {
@@ -102,6 +100,9 @@ func provisioningRequestName(pod *v1.Pod) (string, bool) {
if pod == nil || pod.Annotations == nil {
return "", false
}
provReqName, found := pod.Annotations[ProvisioningRequestPodAnnotationKey]
provReqName, found := pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey]
if !found {
provReqName, found = pod.Annotations[provreqpods.DeprecatedProvisioningRequestPodAnnotationKey]
}
return provReqName, found
}
8 changes: 5 additions & 3 deletions cluster-autoscaler/processors/provreq/pods_filter_test.go
Original file line number Diff line number Diff line change
@@ -24,17 +24,19 @@ import (
"github.com/stretchr/testify/assert"
apiv1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
"k8s.io/client-go/tools/record"
)

func TestProvisioningRequestPodsFilter(t *testing.T) {
prPod1 := BuildTestPod("pr-pod-1", 500, 10)
prPod1.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class"
prPod1.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "pr-class"

prPod2 := BuildTestPod("pr-pod-2", 500, 10)
prPod2.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class-2"
prPod2.Annotations[pods.DeprecatedProvisioningRequestPodAnnotationKey] = "pr-class-2"

pod1 := BuildTestPod("pod-1", 500, 10)
pod2 := BuildTestPod("pod-2", 500, 10)
@@ -91,7 +93,7 @@ func TestEventManager(t *testing.T) {

for i := 0; i < 10; i++ {
prPod := BuildTestPod(fmt.Sprintf("pr-pod-%d", i), 10, 10)
prPod.Annotations[ProvisioningRequestPodAnnotationKey] = "pr-class"
prPod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "pr-class"
unscheduledPods = append(unscheduledPods, prPod)
}
got, err := prFilter.Process(ctx, unscheduledPods)
7 changes: 5 additions & 2 deletions cluster-autoscaler/proposals/provisioning-request.md
Original file line number Diff line number Diff line change
@@ -184,10 +184,13 @@ not required in ProvReq’s template, though can be specified):

```yaml
annotations:
"cluster-autoscaler.kubernetes.io/provisioning-class-name": "provreq-class-name"
"cluster-autoscaler.kubernetes.io/consume-provisioning-request": "provreq-name"
"autoscaling.x-k8s.io/provisioning-class-name": "provreq-class-name"
"autoscaling.x-k8s.io/consume-provisioning-request": "provreq-name"
```
Previous prosoal included annotations with prefix `cluster-autoscaler.kubernetes.io`
but were deprecated as part of API reivew.

If those are provided for the pods that consume the ProvReq with `check-capacity.kubernetes.io` class,
the CA will not provision the capacity, even if it was needed (as some other pods might have been
scheduled on it) and will result in visibility events passed to the ProvReq and pods.
Original file line number Diff line number Diff line change
@@ -19,13 +19,13 @@ package orchestrator
import (
appsv1 "k8s.io/api/apps/v1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1"
"k8s.io/autoscaler/cluster-autoscaler/clusterstate"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/core/scaleup"
"k8s.io/autoscaler/cluster-autoscaler/core/scaleup/orchestrator"
"k8s.io/autoscaler/cluster-autoscaler/estimator"
ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors"
"k8s.io/autoscaler/cluster-autoscaler/processors/provreq"
"k8s.io/autoscaler/cluster-autoscaler/processors/status"
"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
"k8s.io/autoscaler/cluster-autoscaler/utils/taints"
@@ -87,7 +87,7 @@ func (o *WrapperOrchestrator) ScaleUp(

func splitOut(unschedulablePods []*apiv1.Pod) (provReqPods, regularPods []*apiv1.Pod) {
for _, pod := range unschedulablePods {
if _, ok := pod.Annotations[provreq.ProvisioningRequestPodAnnotationKey]; ok {
if _, ok := pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey]; ok {
provReqPods = append(provReqPods, pod)
} else {
regularPods = append(regularPods, pod)
Original file line number Diff line number Diff line change
@@ -22,11 +22,11 @@ import (
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1"
"k8s.io/autoscaler/cluster-autoscaler/clusterstate"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/estimator"
ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors"
"k8s.io/autoscaler/cluster-autoscaler/processors/provreq"
"k8s.io/autoscaler/cluster-autoscaler/processors/status"
"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
"k8s.io/autoscaler/cluster-autoscaler/utils/taints"
@@ -53,7 +53,7 @@ func TestWrapperScaleUp(t *testing.T) {
BuildTestPod("pr-pod-2", 1, 100),
}
for _, pod := range provReqPods {
pod.Annotations[provreq.ProvisioningRequestPodAnnotationKey] = "true"
pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = "true"
}
unschedulablePods := append(regularPods, provReqPods...)
_, err := o.ScaleUp(unschedulablePods, nil, nil, nil, false)
13 changes: 7 additions & 6 deletions cluster-autoscaler/provisioningrequest/pods/pods.go
Original file line number Diff line number Diff line change
@@ -23,15 +23,16 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1beta1"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/provreqwrapper"
"k8s.io/kubernetes/pkg/controller"
)

const (
// ProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request.
ProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request"
// ProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class
ProvisioningClassPodAnnotationKey = "cluster-autoscaler.kubernetes.io/provisioning-class-name"
// DeprecatedProvisioningRequestPodAnnotationKey is a key used to annotate pods consuming provisioning request.
DeprecatedProvisioningRequestPodAnnotationKey = "cluster-autoscaler.kubernetes.io/consume-provisioning-request"
// DeprecatedProvisioningClassPodAnnotationKey is a key used to add annotation about Provisioning Class
DeprecatedProvisioningClassPodAnnotationKey = "cluster-autoscaler.kubernetes.io/provisioning-class-name"
)

// PodsForProvisioningRequest returns a list of pods for which Provisioning
@@ -77,8 +78,8 @@ func populatePodFields(pr *provreqwrapper.ProvisioningRequest, pod *v1.Pod, i, j
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
pod.Annotations[ProvisioningRequestPodAnnotationKey] = pr.Name
pod.Annotations[ProvisioningClassPodAnnotationKey] = pr.Spec.ProvisioningClassName
pod.Annotations[v1beta1.ProvisioningRequestPodAnnotationKey] = pr.Name
pod.Annotations[v1beta1.ProvisioningClassPodAnnotationKey] = pr.Spec.ProvisioningClassName
pod.UID = types.UID(fmt.Sprintf("%s/%s", pod.Namespace, pod.Name))
pod.CreationTimestamp = pr.CreationTimestamp
}
4 changes: 2 additions & 2 deletions cluster-autoscaler/provisioningrequest/pods/pods_test.go
Original file line number Diff line number Diff line change
@@ -42,8 +42,8 @@ func TestPodsForProvisioningRequest(t *testing.T) {
Namespace: "test-namespace",
UID: types.UID(fmt.Sprintf("test-namespace/%s", name)),
Annotations: map[string]string{
ProvisioningRequestPodAnnotationKey: prName,
ProvisioningClassPodAnnotationKey: testProvisioningClassName,
v1beta1.ProvisioningRequestPodAnnotationKey: prName,
v1beta1.ProvisioningClassPodAnnotationKey: testProvisioningClassName,
},
Labels: map[string]string{},
Finalizers: []string{},

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.