From 0e77b7842c283ea42f57c6a7f4c5d56adf4b8bbb Mon Sep 17 00:00:00 2001 From: syedsadath-17 <90619459+sadath-12@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:26:36 +0530 Subject: [PATCH] =?UTF-8?q?feat:=20Support=20PodRequests=20calculations=20?= =?UTF-8?q?for=20initContainers=20with=20restar=E2=80=A6=20(#569)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: sadath-12 Co-authored-by: Jonathan Innis --- kwok/cloudprovider/helpers.go | 4 +- pkg/cloudprovider/fake/instancetype.go | 14 +- .../provisioning/scheduling/suite_test.go | 28 +- pkg/controllers/provisioning/suite_test.go | 239 ++++++- pkg/test/pods.go | 28 +- pkg/utils/resources/resources.go | 77 ++- pkg/utils/resources/suite_test.go | 603 ++++++++++++++++++ 7 files changed, 930 insertions(+), 63 deletions(-) create mode 100644 pkg/utils/resources/suite_test.go diff --git a/kwok/cloudprovider/helpers.go b/kwok/cloudprovider/helpers.go index 286d6e45df..67f140610d 100644 --- a/kwok/cloudprovider/helpers.go +++ b/kwok/cloudprovider/helpers.go @@ -83,7 +83,7 @@ func ConstructInstanceTypes() []*cloudprovider.InstanceType { }, InstanceTypeLabels: labels, } - price := priceFromResources(opts.Resources) + price := PriceFromResources(opts.Resources) opts.Offerings = cloudprovider.Offerings{} for _, zone := range KwokZones { @@ -131,7 +131,7 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType { } } -func priceFromResources(resources v1.ResourceList) float64 { +func PriceFromResources(resources v1.ResourceList) float64 { price := 0.0 for k, v := range resources { switch k { diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index d3b02d6054..8e77322dbf 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -62,11 +62,11 @@ func NewInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType { } if len(options.Offerings) == 0 { options.Offerings = []cloudprovider.Offering{ - {CapacityType: "spot", Zone: "test-zone-1", Price: priceFromResources(options.Resources), Available: true}, - {CapacityType: "spot", Zone: "test-zone-2", Price: priceFromResources(options.Resources), Available: true}, - {CapacityType: "on-demand", Zone: "test-zone-1", Price: priceFromResources(options.Resources), Available: true}, - {CapacityType: "on-demand", Zone: "test-zone-2", Price: priceFromResources(options.Resources), Available: true}, - {CapacityType: "on-demand", Zone: "test-zone-3", Price: priceFromResources(options.Resources), Available: true}, + {CapacityType: "spot", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true}, + {CapacityType: "spot", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true}, + {CapacityType: "on-demand", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true}, + {CapacityType: "on-demand", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true}, + {CapacityType: "on-demand", Zone: "test-zone-3", Price: PriceFromResources(options.Resources), Available: true}, } } if len(options.Architecture) == 0 { @@ -125,7 +125,7 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType { v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", mem)), }, } - price := priceFromResources(opts.Resources) + price := PriceFromResources(opts.Resources) opts.Offerings = []cloudprovider.Offering{ { CapacityType: ct, @@ -173,7 +173,7 @@ type InstanceTypeOptions struct { Resources v1.ResourceList } -func priceFromResources(resources v1.ResourceList) float64 { +func PriceFromResources(resources v1.ResourceList) float64 { price := 0.0 for k, v := range resources { switch k { diff --git a/pkg/controllers/provisioning/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go index 740f01d267..145a874e58 100644 --- a/pkg/controllers/provisioning/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -1663,11 +1663,15 @@ var _ = Context("NodePool", func() { v1.ResourceCPU: resource.MustParse("1"), }, }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceCPU: resource.MustParse("2"), + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("2"), + }, + }, }, }, }) @@ -1684,13 +1688,15 @@ var _ = Context("NodePool", func() { v1.ResourceCPU: resource.MustParse("1"), }, }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceMemory: resource.MustParse("1Ti"), - v1.ResourceCPU: resource.MustParse("2"), + InitContainers: []v1.Container{{ + Resources: v1.ResourceRequirements{ + + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Ti"), + v1.ResourceCPU: resource.MustParse("2"), + }, }, - }, + }}, }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index ad5e985e57..9b9f9ff521 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -52,15 +52,17 @@ import ( . "sigs.k8s.io/karpenter/pkg/test/expectations" ) -var ctx context.Context -var fakeClock *clock.FakeClock -var cluster *state.Cluster -var nodeController controller.Controller -var daemonsetController controller.Controller -var cloudProvider *fake.CloudProvider -var prov *provisioning.Provisioner -var env *test.Environment -var instanceTypeMap map[string]*cloudprovider.InstanceType +var ( + ctx context.Context + fakeClock *clock.FakeClock + cluster *state.Cluster + nodeController controller.Controller + daemonsetController controller.Controller + cloudProvider *fake.CloudProvider + prov *provisioning.Provisioner + env *test.Environment + instanceTypeMap map[string]*cloudprovider.InstanceType +) func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -256,7 +258,8 @@ var _ = Describe("Provisioning", func() { v1.LabelInstanceTypeStable: its[0].Name, }, Finalizers: []string{v1beta1.TerminationFinalizer}, - }}, + }, + }, ) ExpectApplied(ctx, env.Client, node, nodePool) ExpectReconcileSucceeded(ctx, nodeController, client.ObjectKeyFromObject(node)) @@ -342,7 +345,9 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("1.5"), - }}} + }, + }, + } pods := []*v1.Pod{ test.UnschedulablePod(opts), test.UnschedulablePod(opts), @@ -511,7 +516,8 @@ var _ = Describe("Provisioning", func() { ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) pod := test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, - NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}}) + NodeSelector: map[string]string{v1beta1.NodePoolLabelKey: nodePool.Name}, + }) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -540,10 +546,13 @@ var _ = Describe("Provisioning", func() { Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }, }, }}, )) @@ -554,6 +563,159 @@ var _ = Describe("Provisioning", func() { Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when initcontainer comes first", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are smaller than container resources", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are bigger than container resources", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) It("should not schedule if combined max resources are too large for any node", func() { ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{ @@ -561,10 +723,13 @@ var _ = Describe("Provisioning", func() { Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("1Gi")}, Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, }, - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }, }, }}, )) @@ -575,9 +740,12 @@ var _ = Describe("Provisioning", func() { It("should not schedule if initContainer resources are too large", func() { ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{ - InitImage: "pause", - InitResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}, + }, + }, }, }}, )) @@ -726,7 +894,7 @@ var _ = Describe("Provisioning", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod) ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset)) - //Deploy pod + // Deploy pod pod := test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, NodeSelector: map[string]string{ @@ -834,7 +1002,6 @@ var _ = Describe("Provisioning", func() { Expect(node.Labels).To(HaveKeyWithValue("subdomain."+domain+"/test", "test-value")) } }) - }) Context("Taints", func() { It("should schedule pods that tolerate taints", func() { @@ -1662,3 +1829,25 @@ func ExpectNodeClaimRequests(nodeClaim *v1beta1.NodeClaim, resources v1.Resource Expect(v.AsApproximateFloat64()).To(BeNumerically("~", value.AsApproximateFloat64(), 10)) } } + +func AddInstanceResources(instanceTypes []*cloudprovider.InstanceType, resources v1.ResourceList) []*cloudprovider.InstanceType { + opts := fake.InstanceTypeOptions{ + Name: "example", + Architecture: "arch", + Resources: resources, + OperatingSystems: sets.New(string(v1.Linux)), + } + price := fake.PriceFromResources(opts.Resources) + opts.Offerings = []cloudprovider.Offering{ + { + CapacityType: v1beta1.CapacityTypeSpot, + Zone: "test-zone-1", + Price: price, + Available: true, + }, + } + + instanceTypes = append(instanceTypes, fake.NewInstanceType(opts)) + + return instanceTypes +} diff --git a/pkg/test/pods.go b/pkg/test/pods.go index 6bf11173dd..f750882269 100644 --- a/pkg/test/pods.go +++ b/pkg/test/pods.go @@ -32,10 +32,10 @@ import ( type PodOptions struct { metav1.ObjectMeta Image string - InitImage string NodeName string + Overhead v1.ResourceList PriorityClassName string - InitResourceRequirements v1.ResourceRequirements + InitContainers []v1.Container ResourceRequirements v1.ResourceRequirements NodeSelector map[string]string NodeRequirements []v1.NodeSelectorRequirement @@ -71,8 +71,13 @@ type EphemeralVolumeTemplateOptions struct { StorageClassName *string } +const ( + DefaultImage = "public.ecr.aws/eks-distro/kubernetes/pause:3.2" +) + // Pod creates a test pod with defaults that can be overridden by PodOptions. // Overrides are applied in order, with a last write wins semantic. +// nolint:gocyclo func Pod(overrides ...PodOptions) *v1.Pod { options := PodOptions{} for _, opts := range overrides { @@ -81,7 +86,7 @@ func Pod(overrides ...PodOptions) *v1.Pod { } } if options.Image == "" { - options.Image = "public.ecr.aws/eks-distro/kubernetes/pause:3.2" + options.Image = DefaultImage } var volumes []v1.Volume for _, pvc := range options.PersistentVolumeClaims { @@ -164,12 +169,17 @@ func Pod(overrides ...PodOptions) *v1.Pod { if options.Command != nil { p.Spec.Containers[0].Command = options.Command } - if options.InitImage != "" { - p.Spec.InitContainers = []v1.Container{{ - Name: RandomName(), - Image: options.InitImage, - Resources: options.InitResourceRequirements, - }} + if options.Overhead != nil { + p.Spec.Overhead = options.Overhead + } + if options.InitContainers != nil { + for _, init := range options.InitContainers { + init.Name = RandomName() + if init.Image == "" { + init.Image = DefaultImage + } + p.Spec.InitContainers = append(p.Spec.InitContainers, init) + } } return p } diff --git a/pkg/utils/resources/resources.go b/pkg/utils/resources/resources.go index dd88850807..4db2cfe642 100644 --- a/pkg/utils/resources/resources.go +++ b/pkg/utils/resources/resources.go @@ -17,6 +17,7 @@ limitations under the License. package resources import ( + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -95,21 +96,79 @@ func Subtract(lhs, rhs v1.ResourceList) v1.ResourceList { return result } -// Ceiling calculates the max between the sum of container resources and max of initContainers -func Ceiling(pod *v1.Pod) v1.ResourceRequirements { - var resources v1.ResourceRequirements +// podRequests calculates the max between the sum of container resources and max of initContainers along with sidecar feature consideration +// inspired from https://github.com/kubernetes/kubernetes/blob/e2afa175e4077d767745246662170acd86affeaf/pkg/api/v1/resource/helpers.go#L96 +// https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/ +func podRequests(pod *v1.Pod) v1.ResourceList { + requests := v1.ResourceList{} + restartableInitContainerReqs := v1.ResourceList{} + maxInitContainerReqs := v1.ResourceList{} + for _, container := range pod.Spec.Containers { - resources.Requests = MergeInto(resources.Requests, MergeResourceLimitsIntoRequests(container)) - resources.Limits = MergeInto(resources.Limits, container.Resources.Limits) + MergeInto(requests, MergeResourceLimitsIntoRequests(container)) } + for _, container := range pod.Spec.InitContainers { - resources.Requests = MaxResources(resources.Requests, MergeResourceLimitsIntoRequests(container)) - resources.Limits = MaxResources(resources.Limits, container.Resources.Limits) + containerReqs := MergeResourceLimitsIntoRequests(container) + // If the init container's policy is "Always", then we need to add this container's requests to the total requests. We also need to track this container's request as the required requests for other initContainers + if lo.FromPtr(container.RestartPolicy) == v1.ContainerRestartPolicyAlways { + MergeInto(requests, containerReqs) + MergeInto(restartableInitContainerReqs, containerReqs) + maxInitContainerReqs = MaxResources(maxInitContainerReqs, restartableInitContainerReqs) + + } else { + // Else, check whether the current container's resource requests combined with the restartableInitContainer requests are greater than the current max + maxInitContainerReqs = MaxResources(maxInitContainerReqs, Merge(containerReqs, restartableInitContainerReqs)) + } } + // The container's needed requests are the max of all of the container requests combined with native sidecar container requests OR the requests required for a large init containers with native sidecar container requests to run + requests = MaxResources(requests, maxInitContainerReqs) + if pod.Spec.Overhead != nil { - resources.Requests = MergeInto(resources.Requests, pod.Spec.Overhead) + MergeInto(requests, pod.Spec.Overhead) + } + + return requests +} + +// podLimits calculates the max between the sum of container resources and max of initContainers along with sidecar feature consideration +// inspired from https://github.com/kubernetes/kubernetes/blob/e2afa175e4077d767745246662170acd86affeaf/pkg/api/v1/resource/helpers.go#L96 +// https://kubernetes.io/blog/2023/08/25/native-sidecar-containers/ +func podLimits(pod *v1.Pod) v1.ResourceList { + limits := v1.ResourceList{} + restartableInitContainerLimits := v1.ResourceList{} + maxInitContainerLimits := v1.ResourceList{} + + for _, container := range pod.Spec.Containers { + MergeInto(limits, container.Resources.Limits) + } + + for _, container := range pod.Spec.InitContainers { + // If the init container's policy is "Always", then we need to add this container's limits to the total limits. We also need to track this container's limit as the required limits for other initContainers + if lo.FromPtr(container.RestartPolicy) == v1.ContainerRestartPolicyAlways { + MergeInto(limits, container.Resources.Limits) + MergeInto(restartableInitContainerLimits, container.Resources.Limits) + maxInitContainerLimits = MaxResources(maxInitContainerLimits, restartableInitContainerLimits) + } else { + // Else, check whether the current container's resource limits combined with the restartableInitContainer limits are greater than the current max + maxInitContainerLimits = MaxResources(maxInitContainerLimits, Merge(container.Resources.Limits, restartableInitContainerLimits)) + } + } + // The container's needed limits are the max of all of the container limits combined with native sidecar container limits OR the limits required for a large init containers with native sidecar container limits to run + limits = MaxResources(limits, maxInitContainerLimits) + + if pod.Spec.Overhead != nil { + MergeInto(limits, pod.Spec.Overhead) + } + + return limits +} + +func Ceiling(pod *v1.Pod) v1.ResourceRequirements { + return v1.ResourceRequirements{ + Requests: podRequests(pod), + Limits: podLimits(pod), } - return resources } // MaxResources returns the maximum quantities for a given list of resources diff --git a/pkg/utils/resources/suite_test.go b/pkg/utils/resources/suite_test.go new file mode 100644 index 0000000000..c1afe034e4 --- /dev/null +++ b/pkg/utils/resources/suite_test.go @@ -0,0 +1,603 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + . "sigs.k8s.io/karpenter/pkg/test/expectations" + "sigs.k8s.io/karpenter/pkg/utils/resources" + + "sigs.k8s.io/karpenter/pkg/test" +) + +func TestResources(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Resources") +} + +var _ = Describe("Resources", func() { + Context("Resource Calculations", func() { + It("should calculate resource requests based off of the sum of containers and sidecarContainers", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + v1.ResourceMemory: resource.MustParse("3Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + v1.ResourceMemory: resource.MustParse("3Gi"), + }) + }) + It("should calculate resource requests based off of containers, sidecarContainers, initContainers, and overhead", func() { + pod := test.Pod(test.PodOptions{ + Overhead: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }) + }) + It("should calculate resource requests when there is an initContainer after a sidecarContainer that exceeds container resource requests", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }) + }) + It("should calculate resource requests when there is an initContainer after a sidecarContainer that doesn't exceed container resource requests", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }) + }) + Context("Multiple SidecarContainers", func() { + It("should calculate resource requests when there is an initContainer after multiple sidecarContainers that exceeds container resource requests", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20"), v1.ResourceMemory: resource.MustParse("20Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("20"), v1.ResourceMemory: resource.MustParse("20Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("31"), + v1.ResourceMemory: resource.MustParse("31Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("31"), + v1.ResourceMemory: resource.MustParse("31Gi"), + }) + }) + It("should calculate resource requests when there is an initContainer after multiple sidecarContainers that doesn't exceed container resource requests", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("14Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("14Gi"), + }) + }) + It("should calculate resource requests with multiple sidecarContainers when the first initContainer exceeds the sum of all sidecarContainers and container resource requests", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("25"), v1.ResourceMemory: resource.MustParse("25Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("25"), v1.ResourceMemory: resource.MustParse("25Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("25"), + v1.ResourceMemory: resource.MustParse("25Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("25"), + v1.ResourceMemory: resource.MustParse("25Gi"), + }) + }) + It("should calculate resource requests with multiple interspersed sidecarContainers and initContainers", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }, + + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10Gi"), + }) + }) + + }) + Context("Unequal Resource Requests", func() { + It("should calculate resource requests when the first initContainer exceeds cpu for sidecarContainers and containers but not memory", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("25"), v1.ResourceMemory: resource.MustParse("4Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("25"), v1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("25"), + v1.ResourceMemory: resource.MustParse("9Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("25"), + v1.ResourceMemory: resource.MustParse("9Gi"), + }) + }) + It("should calculate resource requests when the first initContainer exceeds memory for sidecarContainers and containers but not cpu", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("3Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("25Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("25Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("25Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("25Gi"), + }) + }) + It("should calculate resource requests when there is an initContainer after a sidecarContainer that exceeds cpu for containers but not memory", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("4Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("4Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }) + }) + It("should calculate resource requests when there is an initContainer after a sidecarContainer that exceeds memory for containers but not cpu", func() { + pod := test.Pod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("4Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + }, + }) + podResources := resources.Ceiling(pod) + ExpectResources(podResources.Requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }) + ExpectResources(podResources.Limits, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("14"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }) + }) + }) + }) + Context("Resource Merging", func() { + It("should merge resource limits into requests if no request exists for the given container", func() { + container := v1.Container{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + } + requests := resources.MergeResourceLimitsIntoRequests(container) + ExpectResources(requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }) + }) + It("should merge resource limits into requests if no request exists for the given sidecarContainer", func() { + container := v1.Container{ + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + } + requests := resources.MergeResourceLimitsIntoRequests(container) + ExpectResources(requests, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }) + }) + }) +})