[workload] Get default resource values from LimitRanges

apis/kueue/webhooks/workload_webhook.go

@@ -19,7 +19,6 @@ package webhooks
 import (
 	"context"
 
-	corev1 "k8s.io/api/core/v1"
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
 	metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -61,30 +60,9 @@ func (w *WorkloadWebhook) Default(ctx context.Context, obj runtime.Object) error
 			podSet.Name = kueue.DefaultPodSetName
 		}
 	}
-	for i := range wl.Spec.PodSets {
-		podSet := &wl.Spec.PodSets[i]
-		setContainersDefaults(podSet.Template.Spec.InitContainers)
-		setContainersDefaults(podSet.Template.Spec.Containers)
-	}
 	return nil
 }
 
-func setContainersDefaults(containers []corev1.Container) {
-	for i := range containers {
-		c := &containers[i]
-		if c.Resources.Limits != nil {
-			if c.Resources.Requests == nil {
-				c.Resources.Requests = make(corev1.ResourceList)
-			}
-			for k, v := range c.Resources.Limits {
-				if _, exists := c.Resources.Requests[k]; !exists {
-					c.Resources.Requests[k] = v.DeepCopy()
-				}
-			}
-		}
-	}
-}
-
 // +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta1-workload,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=workloads;workloads/status,verbs=create;update,versions=v1beta1,name=vworkload.kb.io,admissionReviewVersions=v1
 
 var _ webhook.CustomValidator = &WorkloadWebhook{}

apis/kueue/webhooks/workload_webhook_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
@@ -74,113 +73,6 @@ func TestWorkloadWebhookDefault(t *testing.T) {
 				},
 			},
 		},
-		"copy limits to requests": {
-			wl: kueue.Workload{
-				Spec: kueue.WorkloadSpec{
-					PodSets: []kueue.PodSet{
-						{
-							Name: "foo",
-							Template: corev1.PodTemplateSpec{
-								Spec: corev1.PodSpec{
-									InitContainers: []corev1.Container{
-										{
-											Resources: corev1.ResourceRequirements{
-												Requests: corev1.ResourceList{
-													"cpu": resource.MustParse("1"),
-												},
-												Limits: corev1.ResourceList{
-													"cpu":    resource.MustParse("2"),
-													"memory": resource.MustParse("1Mi"),
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-						{
-							Name: "bar",
-							Template: corev1.PodTemplateSpec{
-								Spec: corev1.PodSpec{
-									Containers: []corev1.Container{
-										{
-											Resources: corev1.ResourceRequirements{
-												Limits: corev1.ResourceList{
-													"cpu":    resource.MustParse("1.5"),
-													"memory": resource.MustParse("5Mi"),
-												},
-											},
-										},
-										{
-											Resources: corev1.ResourceRequirements{
-												Requests: corev1.ResourceList{
-													"cpu": resource.MustParse("1"),
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			wantWl: kueue.Workload{
-				Spec: kueue.WorkloadSpec{
-					PodSets: []kueue.PodSet{
-						{
-							Name: "foo",
-							Template: corev1.PodTemplateSpec{
-								Spec: corev1.PodSpec{
-									InitContainers: []corev1.Container{
-										{
-											Resources: corev1.ResourceRequirements{
-												Requests: corev1.ResourceList{
-													"cpu":    resource.MustParse("1"),
-													"memory": resource.MustParse("1Mi"),
-												},
-												Limits: corev1.ResourceList{
-													"cpu":    resource.MustParse("2"),
-													"memory": resource.MustParse("1Mi"),
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-						{
-							Name: "bar",
-							Template: corev1.PodTemplateSpec{
-								Spec: corev1.PodSpec{
-									Containers: []corev1.Container{
-										{
-											Resources: corev1.ResourceRequirements{
-												Requests: corev1.ResourceList{
-													"cpu":    resource.MustParse("1.5"),
-													"memory": resource.MustParse("5Mi"),
-												},
-												Limits: corev1.ResourceList{
-													"cpu":    resource.MustParse("1.5"),
-													"memory": resource.MustParse("5Mi"),
-												},
-											},
-										},
-										{
-											Resources: corev1.ResourceRequirements{
-												Requests: corev1.ResourceList{
-													"cpu": resource.MustParse("1"),
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {

config/components/rbac/role.yaml

@@ -13,6 +13,14 @@ rules:
   - create
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - limitranges
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

pkg/controller/core/indexer/indexer.go

@@ -20,15 +20,17 @@ import (
 	"context"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 )
 
 const (
-	WorkloadQueueKey        = "spec.queueName"
-	WorkloadClusterQueueKey = "status.admission.clusterQueue"
-	QueueClusterQueueKey    = "spec.clusterQueue"
+	WorkloadQueueKey           = "spec.queueName"
+	WorkloadClusterQueueKey    = "status.admission.clusterQueue"
+	QueueClusterQueueKey       = "spec.clusterQueue"
+	LimitRangeHasContainerType = "spec.hasContainerType"
 )
 
 func IndexQueueClusterQueue(obj client.Object) []string {
@@ -58,6 +60,20 @@ func IndexWorkloadClusterQueue(obj client.Object) []string {
 	return []string{string(wl.Status.Admission.ClusterQueue)}
 }
 
+func IndexLimitRangeHasContainerType(obj client.Object) []string {
+	lr, ok := obj.(*corev1.LimitRange)
+	if !ok {
+		return nil
+	}
+
+	for i := range lr.Spec.Limits {
+		if lr.Spec.Limits[i].Type == corev1.LimitTypeContainer {
+			return []string{"true"}
+		}
+	}
+	return nil
+}
+
 // Setup sets the index with the given fields for core apis.
 func Setup(ctx context.Context, indexer client.FieldIndexer) error {
 	if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadQueueKey, IndexWorkloadQueue); err != nil {
@@ -69,5 +85,8 @@ func Setup(ctx context.Context, indexer client.FieldIndexer) error {
 	if err := indexer.IndexField(ctx, &kueue.LocalQueue{}, QueueClusterQueueKey, IndexQueueClusterQueue); err != nil {
 		return fmt.Errorf("setting index on clusterQueue for localQueue: %w", err)
 	}
+	if err := indexer.IndexField(ctx, &corev1.LimitRange{}, LimitRangeHasContainerType, IndexLimitRangeHasContainerType); err != nil {
+		return fmt.Errorf("setting index on hasContainerType for limitRange: %w", err)
+	}
 	return nil
 }

pkg/controller/core/workload_controller.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
 	nodev1 "k8s.io/api/node/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,7 +36,10 @@ import (
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 	"sigs.k8s.io/kueue/pkg/cache"
 	"sigs.k8s.io/kueue/pkg/constants"
+	"sigs.k8s.io/kueue/pkg/controller/core/indexer"
 	"sigs.k8s.io/kueue/pkg/queue"
+	"sigs.k8s.io/kueue/pkg/util/limitrange"
+	"sigs.k8s.io/kueue/pkg/util/resource"
 	"sigs.k8s.io/kueue/pkg/workload"
 )
 
@@ -119,6 +123,7 @@ func NewWorkloadReconciler(client client.Client, queues *queue.Manager, cache *c
 }
 
 //+kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update
+//+kubebuilder:rbac:groups="",resources=limitranges,verbs=get;list;watch
 //+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update
@@ -200,6 +205,8 @@ func (r *WorkloadReconciler) Create(e event.CreateEvent) bool {
 
 	wlCopy := wl.DeepCopy()
 	handlePodOverhead(r.log, wlCopy, r.client)
+	r.handlePodLimitRange(log, wlCopy)
+	r.handleLimitsToRequests(wlCopy)
 
 	if wl.Status.Admission == nil {
 		if !r.queues.AddOrUpdateWorkload(wlCopy) {
@@ -279,6 +286,8 @@ func (r *WorkloadReconciler) Update(e event.UpdateEvent) bool {
 	wlCopy := wl.DeepCopy()
 	// We do not handle old workload here as it will be deleted or replaced by new one anyway.
 	handlePodOverhead(r.log, wlCopy, r.client)
+	r.handlePodLimitRange(log, wlCopy)
+	r.handleLimitsToRequests(wlCopy)
 
 	switch {
 	case status == finished:
@@ -425,3 +434,50 @@ func handlePodOverhead(log logr.Logger, wl *kueue.Workload, c client.Client) {
 		}
 	}
 }
+
+func (r *WorkloadReconciler) handlePodLimitRange(log logr.Logger, wl *kueue.Workload) {
+	ctx := context.TODO()
+	// get the list of limit ranges
+	var list corev1.LimitRangeList
+	if err := r.client.List(ctx, &list, &client.ListOptions{Namespace: wl.Namespace}, client.MatchingFields{indexer.LimitRangeHasContainerType: "true"}); err != nil {
+		log.Error(err, "Could not list LimitRanges")
+		return
+	}
+
+	if len(list.Items) == 0 {
+		return
+	}
+	summary := limitrange.Summarize(list.Items...)
+	containerLimits, found := summary[corev1.LimitTypeContainer]
+	if !found {
+		return
+	}
+
+	for pi := range wl.Spec.PodSets {
+		pod := &wl.Spec.PodSets[pi].Template.Spec
+		for ci := range pod.InitContainers {
+			res := &pod.InitContainers[ci].Resources
+			res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default)
+			res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest)
+		}
+		for ci := range pod.Containers {
+			res := &pod.Containers[ci].Resources
+			res.Limits = resource.MergeResourceListKeepFirst(res.Limits, containerLimits.Default)
+			res.Requests = resource.MergeResourceListKeepFirst(res.Requests, containerLimits.DefaultRequest)
+		}
+	}
+}
+
+func (r *WorkloadReconciler) handleLimitsToRequests(wl *kueue.Workload) {
+	for pi := range wl.Spec.PodSets {
+		pod := &wl.Spec.PodSets[pi].Template.Spec
+		for ci := range pod.InitContainers {
+			res := &pod.InitContainers[ci].Resources
+			res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits)
+		}
+		for ci := range pod.Containers {
+			res := &pod.Containers[ci].Resources
+			res.Requests = resource.MergeResourceListKeepFirst(res.Requests, res.Limits)
+		}
+	}
+}

pkg/util/limitrange/limitrange.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package limitrange
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	"sigs.k8s.io/kueue/pkg/util/resource"
+)
+
+type Summary map[corev1.LimitType]corev1.LimitRangeItem
+
+// Summarize summarizes the provides ranges by:
+// 1. keeping the lowest values for Max and MaxLimitReqestRatio limits
+// 2. keeping the highest values for Min limits
+// 3. keeping the first encountered values for Default and DefaultRequests
+func Summarize(ranges ...corev1.LimitRange) Summary {
+	ret := make(Summary, 3)
+	for i := range ranges {
+		for _, item := range ranges[i].Spec.Limits {
+			ret[item.Type] = addToSummary(ret[item.Type], item)
+		}
+	}
+	return ret
+}
+
+func addToSummary(summary, item corev1.LimitRangeItem) corev1.LimitRangeItem {
+	summary.Max = resource.MergeResourceListKeepMin(summary.Max, item.Max)
+	summary.Min = resource.MergeResourceListKeepMax(summary.Min, item.Min)
+
+	summary.Default = resource.MergeResourceListKeepFirst(summary.Default, item.Default)
+	summary.DefaultRequest = resource.MergeResourceListKeepFirst(summary.DefaultRequest, item.DefaultRequest)
+
+	summary.MaxLimitRequestRatio = resource.MergeResourceListKeepMin(summary.MaxLimitRequestRatio, item.MaxLimitRequestRatio)
+	return summary
+}