From b1a9fcd98457b212ec35e82462666fdb6a4fe7d0 Mon Sep 17 00:00:00 2001 From: Ardika Bagus <me@ardikabs.com> Date: Wed, 15 Nov 2023 21:44:02 +0700 Subject: [PATCH 1/4] feat: add HorizontalPodAutoscaler support for EnvoyProxy API Signed-off-by: Ardika Bagus <me@ardikabs.com> --- api/v1alpha1/envoyproxy_helpers.go | 24 + api/v1alpha1/envoyproxy_types.go | 8 + api/v1alpha1/kubernetes_helpers.go | 10 + api/v1alpha1/shared_types.go | 37 + api/v1alpha1/zz_generated.deepcopy.go | 38 + .../gateway.envoyproxy.io_envoyproxies.yaml | 682 ++++++++++++++++++ .../templates/infra-manager-rbac.yaml | 9 + internal/infrastructure/kubernetes/infra.go | 10 + .../kubernetes/infra_resource.go | 64 +- .../kubernetes/proxy/resource_provider.go | 50 +- .../proxy/resource_provider_test.go | 84 +++ .../kubernetes/proxy/testdata/hpa/custom.yaml | 25 + .../proxy/testdata/hpa/default.yaml | 18 + .../kubernetes/ratelimit/resource_provider.go | 5 + site/content/en/latest/api/extension_types.md | 19 + 15 files changed, 1078 insertions(+), 5 deletions(-) create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go index 2a0bd91ac0a..d446df9f054 100644 --- a/api/v1alpha1/envoyproxy_helpers.go +++ b/api/v1alpha1/envoyproxy_helpers.go @@ -9,6 +9,11 @@ import ( "fmt" "sort" "strings" + + autoscalingv2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + + "github.com/envoyproxy/gateway/internal/utils/ptr" ) // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings. @@ -37,6 +42,21 @@ func DefaultEnvoyProxyKubeProvider() *EnvoyProxyKubernetesProvider { } } +func DefaultEnvoyProxyHpaMetrics() []autoscalingv2.MetricSpec { + return []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + } +} + // GetEnvoyProxyKubeProvider returns the EnvoyProxyKubernetesProvider of EnvoyProxyProvider or // a default EnvoyProxyKubernetesProvider if unspecified. If EnvoyProxyProvider is not of // type "Kubernetes", a nil EnvoyProxyKubernetesProvider is returned. @@ -64,6 +84,10 @@ func (r *EnvoyProxyProvider) GetEnvoyProxyKubeProvider() *EnvoyProxyKubernetesPr r.Kubernetes.EnvoyService.Type = GetKubernetesServiceType(ServiceTypeLoadBalancer) } + if r.Kubernetes.EnvoyHpa != nil { + r.Kubernetes.EnvoyHpa.setDefault() + } + return r.Kubernetes } diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 4aa7156cb8c..9fcc6584b6b 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -127,6 +127,14 @@ type EnvoyProxyKubernetesProvider struct { // +kubebuilder:validation:XValidation:message="loadBalancerIP can only be set for LoadBalancer type",rule="!has(self.loadBalancerIP) || self.type == 'LoadBalancer'" // +kubebuilder:validation:XValidation:message="loadBalancerIP must be a valid IPv4 address",rule="!has(self.loadBalancerIP) || self.loadBalancerIP.matches(r\"^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\\.|$)){4})\")" EnvoyService *KubernetesServiceSpec `json:"envoyService,omitempty"` + + // EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. + // Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. + // + // +optional + // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="self.maxReplicas > 0" + // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than minReplicas",rule="!has(self.minReplicas) || self.maxReplicas >= self.minReplicas" + EnvoyHpa *KubernetesHorizontalPodAutoscalerSpec `json:"envoyHpa,omitempty"` } // ProxyLogging defines logging parameters for managed proxies. diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 7b6c131e6c9..5a4dd43e7d9 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -106,3 +106,13 @@ func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(imag deployment.Container.Image = DefaultKubernetesContainerImage(image) } } + +func (hpa *KubernetesHorizontalPodAutoscalerSpec) setDefault() { + if hpa.MaxReplicas == 0 { + hpa.MaxReplicas = 1 + } + + if len(hpa.Metrics) == 0 { + hpa.Metrics = DefaultEnvoyProxyHpaMetrics() + } +} diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 1764558d706..17bc7f2bf62 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -7,6 +7,7 @@ package v1alpha1 import ( appv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" ) @@ -275,3 +276,39 @@ const ( // https://github.com/google/re2/wiki/Syntax. StringMatchRegularExpression StringMatchType = "RegularExpression" ) + +// KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment +type KubernetesHorizontalPodAutoscalerSpec struct { + // minReplicas is the lower limit for the number of replicas to which the autoscaler + // can scale down. It defaults to 1 replica. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + // + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + // It cannot be less that minReplicas. It defaults to 1 replica. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + // + // +optional + // +kubebuilder:default=1 + MaxReplicas int32 `json:"maxReplicas,omitempty"` + + // metrics contains the specifications for which to use to calculate the + // desired replica count (the maximum replica count across all metrics will + // be used). + // If left empty, it defaults to being based on CPU utilization with average on 80% usage. + // + // +optional + // + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 88bc53a3ae5..99f5c0866f2 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -11,6 +11,7 @@ package v1alpha1 import ( appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -925,6 +926,11 @@ func (in *EnvoyProxyKubernetesProvider) DeepCopyInto(out *EnvoyProxyKubernetesPr *out = new(KubernetesServiceSpec) (*in).DeepCopyInto(*out) } + if in.EnvoyHpa != nil { + in, out := &in.EnvoyHpa, &out.EnvoyHpa + *out = new(KubernetesHorizontalPodAutoscalerSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyProxyKubernetesProvider. @@ -1428,6 +1434,38 @@ func (in *KubernetesDeploymentSpec) DeepCopy() *KubernetesDeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopyInto(out *KubernetesHorizontalPodAutoscalerSpec) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesHorizontalPodAutoscalerSpec. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopy() *KubernetesHorizontalPodAutoscalerSpec { + if in == nil { + return nil + } + out := new(KubernetesHorizontalPodAutoscalerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesPodSpec) DeepCopyInto(out *KubernetesPodSpec) { *out = *in diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 2b937371634..b4d04d7291e 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5230,6 +5230,688 @@ spec: type: string type: object type: object + envoyHpa: + description: EnvoyHpa defines the Horizontal Pod Autoscaler + settings for Envoy Proxy Deployment. Once the HPA is being + set, Replicas field from EnvoyDeployment will be ignored. + properties: + behavior: + description: behavior configures the scaling behavior + of the target in both Up and Down directions (scaleUp + and scaleDown fields respectively). If not set, the + default HPAScalingRules for scale up and scale down + are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. If not set, the default value is to allow + to scale down to minReplicas pods, with a 300 second + stabilization window (i.e., the highest recommendation + for the last 300sec is used). + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling + Up. If not set, the default value is the higher + of: * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds No stabilization + is used.' + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object + maxReplicas: + default: 1 + description: maxReplicas is the upper limit for the number + of replicas to which the autoscaler can scale up. It + cannot be less that minReplicas. It defaults to 1 replica. + See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + format: int32 + type: integer + metrics: + description: "metrics contains the specifications for + which to use to calculate the desired replica count + (the maximum replica count across all metrics will be + used). If left empty, it defaults to being based on + CPU utilization with average on 80% usage. \n See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior." + items: + description: MetricSpec specifies how to scale based + on a single metric (only `type` and one other matching + field should be set at once). + properties: + containerResource: + description: containerResource refers to a resource + metric (such as those specified in requests and + limits) known to Kubernetes describing a single + container in each pod of the current scale target + (e.g. CPU or memory). Such metrics are built in + to Kubernetes, and have special scaling options + on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature + and can be enabled by the HPAContainerMetrics + feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric + that is not associated with any Kubernetes object. + It allows autoscaling based on information coming + from components running outside of cluster (for + example length of queue in cloud messaging service, + or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing + a single kubernetes object (for example, hits-per-second + on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing + each pod in the current scale target (for example, + transactions-processed-per-second). The values + will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric + (such as those specified in requests and limits) + known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special + scaling options on top of those available to normal + per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It + should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to + a matching field in the object. Note: "ContainerResource" + type is available on when the feature-gate HPAContainerMetrics + is enabled' + type: string + required: + - type + type: object + type: array + minReplicas: + description: minReplicas is the lower limit for the number + of replicas to which the autoscaler can scale down. + It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + format: int32 + type: integer + type: object + x-kubernetes-validations: + - message: maxReplicas must be greater than 0 + rule: self.maxReplicas > 0 + - message: maxReplicas cannot be less than minReplicas + rule: '!has(self.minReplicas) || self.maxReplicas >= self.minReplicas' envoyService: description: EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings diff --git a/charts/gateway-helm/templates/infra-manager-rbac.yaml b/charts/gateway-helm/templates/infra-manager-rbac.yaml index 6f3e5a4677f..3929524f484 100644 --- a/charts/gateway-helm/templates/infra-manager-rbac.yaml +++ b/charts/gateway-helm/templates/infra-manager-rbac.yaml @@ -25,6 +25,15 @@ rules: - get - update - delete +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - get + - update + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index f6b27d26702..145f824e5a0 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,6 +26,7 @@ type ResourceRender interface { Service() (*corev1.Service, error) ConfigMap() (*corev1.ConfigMap, error) Deployment() (*appsv1.Deployment, error) + HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) } // Infra manages the creation and deletion of Kubernetes infrastructure @@ -68,6 +70,10 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to create or update service %s/%s", i.Namespace, r.Name()) } + if err := i.configureHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to configure hpa %s/%s", i.Namespace, r.Name()) + } + return nil } @@ -89,5 +95,9 @@ func (i *Infra) delete(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to delete service %s/%s", i.Namespace, r.Name()) } + if err := i.deleteHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to delete hpa %s/%s", i.Namespace, r.Name()) + } + return nil } diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index af041ad3313..1f4cf5ff5b4 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -10,10 +10,14 @@ import ( "reflect" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" ) @@ -74,8 +78,52 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) Name: deployment.Name, } + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + var opts cmp.Options + if hpa != nil { + opts = append(opts, cmpopts.IgnoreFields(appsv1.DeploymentSpec{}, "Replicas")) + } + return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { - return !reflect.DeepEqual(deployment.Spec, current.Spec) + return !cmp.Equal(current.Spec, deployment.Spec, opts...) + }) +} + +// configureHPA configures HorizontalPodAutoscaler object in the kube api server based on the provided ResourceRender, +// there are two operations on this method: +// 1. if an HPA is defined and it doesn't exist then creates, or otherwise updates +// 2. if an HPA is not defined then delete the object irrespective of its existence +func (i *Infra) configureHPA(ctx context.Context, r ResourceRender) error { + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + // when HorizontalPodAutoscaler is disabled, + // then delete the object in the kube api server if any. + if hpa == nil { + hpa = &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.Client.Delete(ctx, hpa) + } + + current := &autoscalingv2.HorizontalPodAutoscaler{} + key := types.NamespacedName{ + Namespace: hpa.Namespace, + Name: hpa.Name, + } + + return i.Client.CreateOrUpdate(ctx, key, current, hpa, func() bool { + return !cmp.Equal(hpa.Spec, current.Spec) }) } @@ -145,3 +193,17 @@ func (i *Infra) deleteService(ctx context.Context, r ResourceRender) error { return i.Client.Delete(ctx, svc) } + +// deleteHpa deletes the Horizontal Pod Autoscaler associated to its renderer, if it exists. +func (i *Infra) deleteHPA(ctx context.Context, r ResourceRender) error { + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + if hpa == nil { + return nil + } + + return i.Client.Delete(ctx, hpa) +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index bc446f79f9e..f463b1088b3 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -11,6 +11,7 @@ import ( "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -56,7 +57,7 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, }, nil @@ -110,7 +111,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, Annotations: annotations, }, @@ -135,7 +136,7 @@ func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, Data: map[string]string{ @@ -192,7 +193,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: dpLabels, }, Spec: appsv1.DeploymentSpec{ @@ -224,5 +225,46 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, } + // omit the deployment replicas if HPA is being set + if provider.GetEnvoyProxyKubeProvider().EnvoyHpa != nil { + deployment.Spec.Replicas = nil + } + return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + provider := r.infra.GetProxyConfig().GetEnvoyProxyProvider() + if provider.Type != egv1a1.ProviderTypeKubernetes { + return nil, fmt.Errorf("invalid provider type %v for Kubernetes infra manager", provider.Type) + } + + hpaConfig := provider.GetEnvoyProxyKubeProvider().EnvoyHpa + if hpaConfig == nil { + return nil, nil + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling/v2", + Kind: "HorizontalPodAutoscaler", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: r.Name(), + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: r.Name(), + }, + MinReplicas: hpaConfig.MinReplicas, + MaxReplicas: hpaConfig.MaxReplicas, + Metrics: hpaConfig.Metrics, + Behavior: hpaConfig.Behavior, + }, + } + + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 1c784927385..6f5c33833af 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -14,7 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/pointer" "sigs.k8s.io/yaml" @@ -23,6 +25,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/utils/ptr" ) const ( @@ -340,6 +343,7 @@ func TestDeployment(t *testing.T) { if tc.deploy != nil { kube.EnvoyDeployment = tc.deploy } + replace := egv1a1.BootstrapTypeReplace if tc.bootstrap != "" { tc.infra.Proxy.Config.Spec.Bootstrap = &egv1a1.ProxyBootstrap{ @@ -506,3 +510,83 @@ func loadServiceAccount() (*corev1.ServiceAccount, error) { _ = yaml.Unmarshal(saYAML, sa) return sa, nil } + +func TestHorizontalPodAutoscaler(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + infra *ir.Infra + hpa *egv1a1.KubernetesHorizontalPodAutoscalerSpec + }{ + { + caseName: "default", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{}, + }, + { + caseName: "custom", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: 10, + Metrics: []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](60), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](70), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + provider := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() + provider.Kubernetes = egv1a1.DefaultEnvoyProxyKubeProvider() + + if tc.hpa != nil { + provider.Kubernetes.EnvoyHpa = tc.hpa + } + + provider.GetEnvoyProxyKubeProvider() + + r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra()) + hpa, err := r.HorizontalPodAutoscaler() + require.NoError(t, err) + + want, err := loadHPA(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, want, hpa) + }) + } +} + +func loadHPA(caseName string) (*autoscalingv2.HorizontalPodAutoscaler, error) { + hpaYAML, err := os.ReadFile(fmt.Sprintf("testdata/hpa/%s.yaml", caseName)) + if err != nil { + return nil, err + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{} + _ = yaml.Unmarshal(hpaYAML, hpa) + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml new file mode 100644 index 00000000000..17171f8abb3 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml @@ -0,0 +1,25 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 60 + type: Utilization + type: Resource + - resource: + name: memory + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 5 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml new file mode 100644 index 00000000000..d11b7e47636 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + metrics: + - resource: + name: cpu + target: + averageUtilization: 80 + type: Utilization + type: Resource + maxReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index f39961795cc..bb7f2ee598d 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -7,6 +7,7 @@ package ratelimit import ( appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -202,3 +203,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + return nil, nil +} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 7379bf3cef7..5efbbe8a865 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -647,6 +647,7 @@ _Appears in:_ | --- | --- | | `envoyDeployment` _[KubernetesDeploymentSpec](#kubernetesdeploymentspec)_ | EnvoyDeployment defines the desired state of the Envoy deployment resource. If unspecified, default settings for the managed Envoy deployment resource are applied. | | `envoyService` _[KubernetesServiceSpec](#kubernetesservicespec)_ | EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings for the managed Envoy service resource are applied. | +| `envoyHpa` _[KubernetesHorizontalPodAutoscalerSpec](#kuberneteshorizontalpodautoscalerspec)_ | EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. | #### EnvoyProxyProvider @@ -976,6 +977,24 @@ _Appears in:_ | `initContainers` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#container-v1-core) array_ | List of initialization containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +#### KubernetesHorizontalPodAutoscalerSpec + + + +KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment + +_Appears in:_ +- [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider) + +| Field | Description | +| --- | --- | +| `minReplicas` _integer_ | minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | +| `maxReplicas` _integer_ | maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | +| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#metricspec-v2-autoscaling) array_ | metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). If left empty, it defaults to being based on CPU utilization with average on 80% usage. + See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | +| `behavior` _[HorizontalPodAutoscalerBehavior](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#horizontalpodautoscalerbehavior-v2-autoscaling)_ | behavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | + + #### KubernetesPodSpec From d4d16f564a99bd829f6503af103c14ea29b7fdba Mon Sep 17 00:00:00 2001 From: Ardika Bagus <me@ardikabs.com> Date: Thu, 30 Nov 2023 19:20:58 +0700 Subject: [PATCH 2/4] chore: change maxReplicas with pointer Signed-off-by: Ardika Bagus <me@ardikabs.com> --- api/v1alpha1/envoyproxy_types.go | 3 ++- api/v1alpha1/kubernetes_helpers.go | 4 ---- api/v1alpha1/shared_types.go | 3 +-- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ .../crds/generated/gateway.envoyproxy.io_envoyproxies.yaml | 5 +++-- .../infrastructure/kubernetes/proxy/resource_provider.go | 3 ++- .../kubernetes/proxy/resource_provider_test.go | 2 +- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 9fcc6584b6b..e11da79d850 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -132,7 +132,8 @@ type EnvoyProxyKubernetesProvider struct { // Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. // // +optional - // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="self.maxReplicas > 0" + // +kubebuilder:validation:XValidation:message="minReplicas must be greater than 0",rule="!has(self.minReplicas) || self.minReplicas > 0" + // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="!has(self.maxReplicas) || self.maxReplicas > 0" // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than minReplicas",rule="!has(self.minReplicas) || self.maxReplicas >= self.minReplicas" EnvoyHpa *KubernetesHorizontalPodAutoscalerSpec `json:"envoyHpa,omitempty"` } diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 5a4dd43e7d9..90c75873cbb 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -108,10 +108,6 @@ func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(imag } func (hpa *KubernetesHorizontalPodAutoscalerSpec) setDefault() { - if hpa.MaxReplicas == 0 { - hpa.MaxReplicas = 1 - } - if len(hpa.Metrics) == 0 { hpa.Metrics = DefaultEnvoyProxyHpaMetrics() } diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 17bc7f2bf62..bcc406dba69 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -291,8 +291,7 @@ type KubernetesHorizontalPodAutoscalerSpec struct { // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec // // +optional - // +kubebuilder:default=1 - MaxReplicas int32 `json:"maxReplicas,omitempty"` + MaxReplicas *int32 `json:"maxReplicas,omitempty"` // metrics contains the specifications for which to use to calculate the // desired replica count (the maximum replica count across all metrics will diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 99f5c0866f2..2654e070a27 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1442,6 +1442,11 @@ func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopyInto(out *KubernetesHor *out = new(int32) **out = **in } + if in.MaxReplicas != nil { + in, out := &in.MaxReplicas, &out.MaxReplicas + *out = new(int32) + **out = **in + } if in.Metrics != nil { in, out := &in.Metrics, &out.Metrics *out = make([]v2.MetricSpec, len(*in)) diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index b4d04d7291e..d0bd299aacc 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5364,7 +5364,6 @@ spec: type: object type: object maxReplicas: - default: 1 description: maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. It defaults to 1 replica. @@ -5908,8 +5907,10 @@ spec: type: integer type: object x-kubernetes-validations: + - message: minReplicas must be greater than 0 + rule: '!has(self.minReplicas) || self.minReplicas > 0' - message: maxReplicas must be greater than 0 - rule: self.maxReplicas > 0 + rule: '!has(self.maxReplicas) || self.maxReplicas > 0' - message: maxReplicas cannot be less than minReplicas rule: '!has(self.minReplicas) || self.maxReplicas >= self.minReplicas' envoyService: diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index f463b1088b3..6ab47c0b334 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi" @@ -260,7 +261,7 @@ func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPod Name: r.Name(), }, MinReplicas: hpaConfig.MinReplicas, - MaxReplicas: hpaConfig.MaxReplicas, + MaxReplicas: ptr.Deref[int32](hpaConfig.MaxReplicas, 1), Metrics: hpaConfig.Metrics, Behavior: hpaConfig.Behavior, }, diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 6f5c33833af..f1b295475f8 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -530,7 +530,7 @@ func TestHorizontalPodAutoscaler(t *testing.T) { infra: newTestInfra(), hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ MinReplicas: ptr.To[int32](5), - MaxReplicas: 10, + MaxReplicas: ptr.To[int32](10), Metrics: []autoscalingv2.MetricSpec{ { Resource: &autoscalingv2.ResourceMetricSource{ From f5bdab33a3341b7ac24a4914be1e695ef0f0e035 Mon Sep 17 00:00:00 2001 From: Ardika Bagus <me@ardikabs.com> Date: Sat, 2 Dec 2023 22:06:49 +0700 Subject: [PATCH 3/4] chore: add celvalidation for envoyHpa Signed-off-by: Ardika Bagus <me@ardikabs.com> --- api/v1alpha1/envoyproxy_types.go | 2 +- api/v1alpha1/shared_types.go | 10 +-- .../gateway.envoyproxy.io_envoyproxies.yaml | 21 ++--- .../kubernetes/infra_resource.go | 5 +- .../proxy/resource_provider_test.go | 5 +- site/content/en/latest/api/extension_types.md | 9 +-- test/cel-validation/envoyproxy_test.go | 80 +++++++++++++++++++ 7 files changed, 104 insertions(+), 28 deletions(-) diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index e11da79d850..acf34417c76 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -134,7 +134,7 @@ type EnvoyProxyKubernetesProvider struct { // +optional // +kubebuilder:validation:XValidation:message="minReplicas must be greater than 0",rule="!has(self.minReplicas) || self.minReplicas > 0" // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="!has(self.maxReplicas) || self.maxReplicas > 0" - // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than minReplicas",rule="!has(self.minReplicas) || self.maxReplicas >= self.minReplicas" + // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than or equal to minReplicas",rule="!has(self.minReplicas) || self.maxReplicas > self.minReplicas" EnvoyHpa *KubernetesHorizontalPodAutoscalerSpec `json:"envoyHpa,omitempty"` } diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index bcc406dba69..e6d19f960ae 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -278,20 +278,18 @@ const ( ) // KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment +// See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec type KubernetesHorizontalPodAutoscalerSpec struct { // minReplicas is the lower limit for the number of replicas to which the autoscaler // can scale down. It defaults to 1 replica. - // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec // // +optional MinReplicas *int32 `json:"minReplicas,omitempty"` // maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. - // It cannot be less that minReplicas. It defaults to 1 replica. - // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + // It cannot be less that minReplicas. // - // +optional - MaxReplicas *int32 `json:"maxReplicas,omitempty"` + MaxReplicas *int32 `json:"maxReplicas"` // metrics contains the specifications for which to use to calculate the // desired replica count (the maximum replica count across all metrics will @@ -299,8 +297,6 @@ type KubernetesHorizontalPodAutoscalerSpec struct { // If left empty, it defaults to being based on CPU utilization with average on 80% usage. // // +optional - // - // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` // behavior configures the scaling behavior of the target diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index d0bd299aacc..7bff9e4149a 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5366,16 +5366,15 @@ spec: maxReplicas: description: maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It - cannot be less that minReplicas. It defaults to 1 replica. - See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + cannot be less that minReplicas. format: int32 type: integer metrics: - description: "metrics contains the specifications for - which to use to calculate the desired replica count - (the maximum replica count across all metrics will be - used). If left empty, it defaults to being based on - CPU utilization with average on 80% usage. \n See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior." + description: metrics contains the specifications for which + to use to calculate the desired replica count (the maximum + replica count across all metrics will be used). If left + empty, it defaults to being based on CPU utilization + with average on 80% usage. items: description: MetricSpec specifies how to scale based on a single metric (only `type` and one other matching @@ -5902,17 +5901,19 @@ spec: minReplicas: description: minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. - It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + It defaults to 1 replica. format: int32 type: integer + required: + - maxReplicas type: object x-kubernetes-validations: - message: minReplicas must be greater than 0 rule: '!has(self.minReplicas) || self.minReplicas > 0' - message: maxReplicas must be greater than 0 rule: '!has(self.maxReplicas) || self.maxReplicas > 0' - - message: maxReplicas cannot be less than minReplicas - rule: '!has(self.minReplicas) || self.maxReplicas >= self.minReplicas' + - message: maxReplicas cannot be less than or equal to minReplicas + rule: '!has(self.minReplicas) || self.maxReplicas > self.minReplicas' envoyService: description: EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 1f4cf5ff5b4..248cbe11af4 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -9,15 +9,14 @@ import ( "context" "reflect" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" ) diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index f1b295475f8..f4603fc370c 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -16,7 +16,6 @@ import ( appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/pointer" "sigs.k8s.io/yaml" @@ -523,7 +522,9 @@ func TestHorizontalPodAutoscaler(t *testing.T) { { caseName: "default", infra: newTestInfra(), - hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{}, + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MaxReplicas: ptr.To[int32](1), + }, }, { caseName: "custom", diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 5efbbe8a865..4fa146d29a9 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -981,17 +981,16 @@ _Appears in:_ -KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment +KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec _Appears in:_ - [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider) | Field | Description | | --- | --- | -| `minReplicas` _integer_ | minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | -| `maxReplicas` _integer_ | maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | -| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#metricspec-v2-autoscaling) array_ | metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). If left empty, it defaults to being based on CPU utilization with average on 80% usage. - See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | +| `minReplicas` _integer_ | minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 replica. | +| `maxReplicas` _integer_ | maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. | +| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#metricspec-v2-autoscaling) array_ | metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). If left empty, it defaults to being based on CPU utilization with average on 80% usage. | | `behavior` _[HorizontalPodAutoscalerBehavior](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#horizontalpodautoscalerbehavior-v2-autoscaling)_ | behavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | diff --git a/test/cel-validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go index ac949ff971d..86b5a8ddc5e 100644 --- a/test/cel-validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -419,6 +419,86 @@ func TestEnvoyProxyProvider(t *testing.T) { }, wantErrors: []string{}, }, + { + desc: "ProxyHpa-maxReplicas-is-required", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{}, + }, + }, + } + }, + wantErrors: []string{"spec.provider.kubernetes.envoyHpa.maxReplicas: Required value"}, + }, + { + desc: "ProxyHpa-minReplicas-less-than-0", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](-1), + MaxReplicas: ptr.To[int32](2), + }, + }, + }, + } + }, + wantErrors: []string{"minReplicas must be greater than 0"}, + }, + { + desc: "ProxyHpa-maxReplicas-less-than-0", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MaxReplicas: ptr.To[int32](-1), + }, + }, + }, + } + }, + wantErrors: []string{"maxReplicas must be greater than 0"}, + }, + { + desc: "ProxyHpa-maxReplicas-less-than-minReplicas", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](2), + }, + }, + }, + } + }, + wantErrors: []string{"maxReplicas cannot be less than or equal to minReplicas"}, + }, + { + desc: "ProxyHpa-valid", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](10), + }, + }, + }, + } + }, + }, } for _, tc := range cases { From 3ad932e31e1cef87b2d33c57e2caa9c2d42fe8de Mon Sep 17 00:00:00 2001 From: Ardika Bagus <me@ardikabs.com> Date: Wed, 6 Dec 2023 11:21:01 +0700 Subject: [PATCH 4/4] chore: move hpa cleanup into deleteHPA Signed-off-by: Ardika Bagus <me@ardikabs.com> --- internal/infrastructure/kubernetes/infra.go | 4 +-- .../kubernetes/infra_resource.go | 32 +++++++------------ 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index 145f824e5a0..0d1f6e18c03 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -70,8 +70,8 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to create or update service %s/%s", i.Namespace, r.Name()) } - if err := i.configureHPA(ctx, r); err != nil { - return errors.Wrapf(err, "failed to configure hpa %s/%s", i.Namespace, r.Name()) + if err := i.createOrUpdateHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to create or update hpa %s/%s", i.Namespace, r.Name()) } return nil diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 248cbe11af4..d0e21be628e 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -92,27 +92,19 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) }) } -// configureHPA configures HorizontalPodAutoscaler object in the kube api server based on the provided ResourceRender, -// there are two operations on this method: -// 1. if an HPA is defined and it doesn't exist then creates, or otherwise updates -// 2. if an HPA is not defined then delete the object irrespective of its existence -func (i *Infra) configureHPA(ctx context.Context, r ResourceRender) error { +// createOrUpdateHPA creates HorizontalPodAutoscaler object in the kube api server based on +// the provided ResourceRender, if it doesn't exist and updates it if it does, +// and delete hpa if not set. +func (i *Infra) createOrUpdateHPA(ctx context.Context, r ResourceRender) error { hpa, err := r.HorizontalPodAutoscaler() if err != nil { return err } - // when HorizontalPodAutoscaler is disabled, + // when HorizontalPodAutoscaler is not set, // then delete the object in the kube api server if any. if hpa == nil { - hpa = &autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: r.Name(), - }, - } - - return i.Client.Delete(ctx, hpa) + return i.deleteHPA(ctx, r) } current := &autoscalingv2.HorizontalPodAutoscaler{} @@ -195,13 +187,11 @@ func (i *Infra) deleteService(ctx context.Context, r ResourceRender) error { // deleteHpa deletes the Horizontal Pod Autoscaler associated to its renderer, if it exists. func (i *Infra) deleteHPA(ctx context.Context, r ResourceRender) error { - hpa, err := r.HorizontalPodAutoscaler() - if err != nil { - return err - } - - if hpa == nil { - return nil + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, } return i.Client.Delete(ctx, hpa)