Merge pull request #1882 from csrwng/resource_overrides

Add annotation to allow resource requests overrides

api/v1alpha1/hostedcluster_types.go

@@ -118,6 +118,13 @@ const (
 	// removed when deleting the corresponding HostedCluster. If set to "true", resources created on the cloud provider during the life
 	// of the cluster will be removed, including image registry storage, ingress dns records, load balancers, and persistent storage.
 	CleanupCloudResourcesAnnotation = "hypershift.openshift.io/cleanup-cloud-resources"
+
+	// ResourceRequestOverrideAnnotationPrefix is a prefix for an annotation to override resource requests for a particular deployment/container
+	// in a hosted control plane. The format of the annotation is:
+	// resource-request-override.hypershift.openshift.io/[deployment-name].[container-name]: [resource-type-1]=[value1],[resource-type-2]=[value2],...
+	// For example, to override the memory and cpu request for the Kubernetes APIServer:
+	// resource-request-override.hypershift.openshift.io/kube-apiserver.kube-apiserver: memory=3Gi,cpu=2000m
+	ResourceRequestOverrideAnnotationPrefix = "resource-request-override.hypershift.openshift.io"
 )
 
 // HostedClusterSpec is the desired behavior of a HostedCluster.

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -1469,7 +1469,8 @@ func reconcileHostedControlPlane(hcp *hyperv1.HostedControlPlane, hcluster *hype
 
 	// All annotations on the HostedCluster with this special prefix are copied
 	for key, val := range hcluster.Annotations {
-		if strings.HasPrefix(key, hyperv1.IdentityProviderOverridesAnnotationPrefix) {
+		if strings.HasPrefix(key, hyperv1.IdentityProviderOverridesAnnotationPrefix) ||
+			strings.HasPrefix(key, hyperv1.ResourceRequestOverrideAnnotationPrefix) {
 			hcp.Annotations[key] = val
 		}
 	}

support/config/deployment.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openshift/hypershift/support/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -43,6 +44,7 @@ type DeploymentConfig struct {
 	ReadinessProbes           ReadinessProbes       `json:"readinessProbes"`
 	Resources                 ResourcesSpec         `json:"resources"`
 	DebugDeployments          sets.String           `json:"debugDeployments"`
+	ResourceRequestOverrides  ResourceOverrides     `json:"resourceRequestOverrides"`
 }
 
 func (c *DeploymentConfig) SetContainerResourcesIfPresent(container *corev1.Container) {
@@ -106,10 +108,10 @@ func (c *DeploymentConfig) ApplyTo(deployment *appsv1.Deployment) {
 	c.Scheduling.ApplyTo(&deployment.Spec.Template.Spec)
 	c.AdditionalLabels.ApplyTo(&deployment.Spec.Template.ObjectMeta)
 	c.SecurityContexts.ApplyTo(&deployment.Spec.Template.Spec)
-	c.Resources.ApplyTo(&deployment.Spec.Template.Spec)
 	c.LivenessProbes.ApplyTo(&deployment.Spec.Template.Spec)
 	c.ReadinessProbes.ApplyTo(&deployment.Spec.Template.Spec)
 	c.Resources.ApplyTo(&deployment.Spec.Template.Spec)
+	c.ResourceRequestOverrides.ApplyRequestsTo(deployment.Name, &deployment.Spec.Template.Spec)
 	c.AdditionalAnnotations.ApplyTo(&deployment.Spec.Template.ObjectMeta)
 }
 
@@ -118,10 +120,10 @@ func (c *DeploymentConfig) ApplyToDaemonSet(daemonset *appsv1.DaemonSet) {
 	c.Scheduling.ApplyTo(&daemonset.Spec.Template.Spec)
 	c.AdditionalLabels.ApplyTo(&daemonset.Spec.Template.ObjectMeta)
 	c.SecurityContexts.ApplyTo(&daemonset.Spec.Template.Spec)
-	c.Resources.ApplyTo(&daemonset.Spec.Template.Spec)
 	c.LivenessProbes.ApplyTo(&daemonset.Spec.Template.Spec)
 	c.ReadinessProbes.ApplyTo(&daemonset.Spec.Template.Spec)
 	c.Resources.ApplyTo(&daemonset.Spec.Template.Spec)
+	c.ResourceRequestOverrides.ApplyRequestsTo(daemonset.Name, &daemonset.Spec.Template.Spec)
 	c.AdditionalAnnotations.ApplyTo(&daemonset.Spec.Template.ObjectMeta)
 }
 
@@ -130,10 +132,10 @@ func (c *DeploymentConfig) ApplyToStatefulSet(sts *appsv1.StatefulSet) {
 	c.Scheduling.ApplyTo(&sts.Spec.Template.Spec)
 	c.AdditionalLabels.ApplyTo(&sts.Spec.Template.ObjectMeta)
 	c.SecurityContexts.ApplyTo(&sts.Spec.Template.Spec)
-	c.Resources.ApplyTo(&sts.Spec.Template.Spec)
 	c.LivenessProbes.ApplyTo(&sts.Spec.Template.Spec)
 	c.ReadinessProbes.ApplyTo(&sts.Spec.Template.Spec)
 	c.Resources.ApplyTo(&sts.Spec.Template.Spec)
+	c.ResourceRequestOverrides.ApplyRequestsTo(sts.Name, &sts.Spec.Template.Spec)
 	c.AdditionalAnnotations.ApplyTo(&sts.Spec.Template.ObjectMeta)
 }
 
@@ -286,11 +288,53 @@ func (c *DeploymentConfig) SetDefaults(hcp *hyperv1.HostedControlPlane, multiZon
 	}
 	c.DebugDeployments = debugDeployments(hcp)
 
+	c.ResourceRequestOverrides = resourceRequestOverrides(hcp)
+
 	c.setLocation(hcp, multiZoneSpreadLabels)
 	// TODO (alberto): make this private, atm is needed for the konnectivity agent daemonset.
 	c.SetReleaseImageAnnotation(hcp.Spec.ReleaseImage)
 }
 
+func resourceRequestOverrides(hcp *hyperv1.HostedControlPlane) ResourceOverrides {
+	result := ResourceOverrides{}
+	for key, value := range hcp.Annotations {
+		if strings.HasPrefix(key, hyperv1.ResourceRequestOverrideAnnotationPrefix+"/") {
+			result = parseResourceRequestOverrideAnnotation(key, value, result)
+		}
+	}
+	return result
+}
+
+func parseResourceRequestOverrideAnnotation(key, value string, overrides ResourceOverrides) ResourceOverrides {
+	keyParts := strings.SplitN(key, "/", 2)
+	deploymentContainerParts := strings.SplitN(keyParts[1], ".", 2)
+	deployment, container := deploymentContainerParts[0], deploymentContainerParts[1]
+	resourceRequests := strings.Split(value, ",")
+	spec, exists := overrides[deployment]
+	if !exists {
+		spec = ResourcesSpec{}
+	}
+	requirements, exists := spec[container]
+	if !exists {
+		requirements = corev1.ResourceRequirements{}
+	}
+	if requirements.Requests == nil {
+		requirements.Requests = corev1.ResourceList{}
+	}
+	for _, request := range resourceRequests {
+		requestParts := strings.SplitN(request, "=", 2)
+		quantity, err := resource.ParseQuantity(requestParts[1])
+		if err != nil {
+			// Skip this request if invalid
+			continue
+		}
+		requirements.Requests[corev1.ResourceName(requestParts[0])] = quantity
+	}
+	spec[container] = requirements
+	overrides[deployment] = spec
+	return overrides
+}
+
 // debugDeployments returns a set of deployments to debug based on the
 // debugDeploymentsAnnotation value, indicating the deployment should be considered to
 // be in development mode.

support/config/deployment_test.go

@@ -7,6 +7,7 @@ import (
 	. "github.com/onsi/gomega"
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -283,3 +284,111 @@ func TestSetLocation(t *testing.T) {
 	}
 	g.Expect(expected.Scheduling.Affinity.PodAntiAffinity).To(BeEquivalentTo(cfg.Scheduling.Affinity.PodAntiAffinity))
 }
+
+func TestResourceRequestOverrides(t *testing.T) {
+	tests := []struct {
+		name        string
+		annotations map[string]string
+		expected    ResourceOverrides
+	}{
+		{
+			name:     "no annotations",
+			expected: ResourceOverrides{},
+		},
+		{
+			name: "override memory",
+			annotations: map[string]string{
+				"random": "foobar",
+				"resource-request-override.hypershift.openshift.io/my-deployment.main-container": "memory=1Gi",
+			},
+			expected: ResourceOverrides{
+				"my-deployment": ResourcesSpec{
+					"main-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceMemory: resource.MustParse("1Gi"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "override memory and cpu",
+			annotations: map[string]string{
+				"random": "foobar",
+				"resource-request-override.hypershift.openshift.io/my-deployment.main-container": "memory=1Gi,cpu=500m",
+			},
+			expected: ResourceOverrides{
+				"my-deployment": ResourcesSpec{
+					"main-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceMemory: resource.MustParse("1Gi"),
+							corev1.ResourceCPU:    resource.MustParse("500m"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "overrides for different containers",
+			annotations: map[string]string{
+				"random": "foobar",
+				"resource-request-override.hypershift.openshift.io/my-deployment.main-container": "memory=1Gi",
+				"resource-request-override.hypershift.openshift.io/my-deployment.init-container": "cpu=500m",
+			},
+			expected: ResourceOverrides{
+				"my-deployment": ResourcesSpec{
+					"main-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceMemory: resource.MustParse("1Gi"),
+						},
+					},
+					"init-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceCPU: resource.MustParse("500m"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "overrides for different deployments",
+			annotations: map[string]string{
+				"random": "foobar",
+				"resource-request-override.hypershift.openshift.io/my-deployment.main-container":     "memory=1Gi",
+				"resource-request-override.hypershift.openshift.io/my-deployment.init-container":     "cpu=500m",
+				"resource-request-override.hypershift.openshift.io/second-deployment.main-container": "cpu=300m,memory=500Mi",
+			},
+			expected: ResourceOverrides{
+				"my-deployment": ResourcesSpec{
+					"main-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceMemory: resource.MustParse("1Gi"),
+						},
+					},
+					"init-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceCPU: resource.MustParse("500m"),
+						},
+					},
+				},
+				"second-deployment": ResourcesSpec{
+					"main-container": corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceMemory: resource.MustParse("500Mi"),
+							corev1.ResourceCPU:    resource.MustParse("300m"),
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			g := NewGomegaWithT(t)
+			hcp := &hyperv1.HostedControlPlane{}
+			hcp.Annotations = test.annotations
+			actual := resourceRequestOverrides(hcp)
+			g.Expect(actual).To(Equal(test.expected))
+		})
+	}
+}

support/config/resources.go

@@ -18,3 +18,28 @@ func (s ResourcesSpec) ApplyTo(podSpec *corev1.PodSpec) {
 		}
 	}
 }
+
+func (s ResourcesSpec) ApplyRequestsOverrideTo(podSpec *corev1.PodSpec) {
+	for i, c := range podSpec.InitContainers {
+		if res, ok := s[c.Name]; ok {
+			for name, value := range res.Requests {
+				podSpec.InitContainers[i].Resources.Requests[name] = value
+			}
+		}
+	}
+	for i, c := range podSpec.Containers {
+		if res, ok := s[c.Name]; ok {
+			for name, value := range res.Requests {
+				podSpec.Containers[i].Resources.Requests[name] = value
+			}
+		}
+	}
+}
+
+type ResourceOverrides map[string]ResourcesSpec
+
+func (o ResourceOverrides) ApplyRequestsTo(name string, podSpec *corev1.PodSpec) {
+	if spec, exists := o[name]; exists {
+		spec.ApplyRequestsOverrideTo(podSpec)
+	}
+}