diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go index 2a0bd91ac0a0..1dfdef1ba8c3 100644 --- a/api/v1alpha1/envoyproxy_helpers.go +++ b/api/v1alpha1/envoyproxy_helpers.go @@ -9,6 +9,10 @@ import ( "fmt" "sort" "strings" + + "github.com/envoyproxy/gateway/internal/utils/ptr" + autoscalingv2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" ) // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings. @@ -37,6 +41,21 @@ func DefaultEnvoyProxyKubeProvider() *EnvoyProxyKubernetesProvider { } } +func DefaultEnvoyProxyHpaMetrics() []autoscalingv2.MetricSpec { + return []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + } +} + // GetEnvoyProxyKubeProvider returns the EnvoyProxyKubernetesProvider of EnvoyProxyProvider or // a default EnvoyProxyKubernetesProvider if unspecified. If EnvoyProxyProvider is not of // type "Kubernetes", a nil EnvoyProxyKubernetesProvider is returned. @@ -64,6 +83,10 @@ func (r *EnvoyProxyProvider) GetEnvoyProxyKubeProvider() *EnvoyProxyKubernetesPr r.Kubernetes.EnvoyService.Type = GetKubernetesServiceType(ServiceTypeLoadBalancer) } + if r.Kubernetes.EnvoyHpa != nil { + r.Kubernetes.EnvoyHpa.setDefault() + } + return r.Kubernetes } diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 4aa7156cb8c3..9fcc6584b6b3 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -127,6 +127,14 @@ type EnvoyProxyKubernetesProvider struct { // +kubebuilder:validation:XValidation:message="loadBalancerIP can only be set for LoadBalancer type",rule="!has(self.loadBalancerIP) || self.type == 'LoadBalancer'" // +kubebuilder:validation:XValidation:message="loadBalancerIP must be a valid IPv4 address",rule="!has(self.loadBalancerIP) || self.loadBalancerIP.matches(r\"^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\\.|$)){4})\")" EnvoyService *KubernetesServiceSpec `json:"envoyService,omitempty"` + + // EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. + // Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. + // + // +optional + // +kubebuilder:validation:XValidation:message="maxReplicas must be greater than 0",rule="self.maxReplicas > 0" + // +kubebuilder:validation:XValidation:message="maxReplicas cannot be less than minReplicas",rule="!has(self.minReplicas) || self.maxReplicas >= self.minReplicas" + EnvoyHpa *KubernetesHorizontalPodAutoscalerSpec `json:"envoyHpa,omitempty"` } // ProxyLogging defines logging parameters for managed proxies. diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 7b6c131e6c97..5a4dd43e7d9e 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -106,3 +106,13 @@ func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(imag deployment.Container.Image = DefaultKubernetesContainerImage(image) } } + +func (hpa *KubernetesHorizontalPodAutoscalerSpec) setDefault() { + if hpa.MaxReplicas == 0 { + hpa.MaxReplicas = 1 + } + + if len(hpa.Metrics) == 0 { + hpa.Metrics = DefaultEnvoyProxyHpaMetrics() + } +} diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 1764558d706f..17bc7f2bf627 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -7,6 +7,7 @@ package v1alpha1 import ( appv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" ) @@ -275,3 +276,39 @@ const ( // https://github.com/google/re2/wiki/Syntax. StringMatchRegularExpression StringMatchType = "RegularExpression" ) + +// KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment +type KubernetesHorizontalPodAutoscalerSpec struct { + // minReplicas is the lower limit for the number of replicas to which the autoscaler + // can scale down. It defaults to 1 replica. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + // + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + // It cannot be less that minReplicas. It defaults to 1 replica. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + // + // +optional + // +kubebuilder:default=1 + MaxReplicas int32 `json:"maxReplicas,omitempty"` + + // metrics contains the specifications for which to use to calculate the + // desired replica count (the maximum replica count across all metrics will + // be used). + // If left empty, it defaults to being based on CPU utilization with average on 80% usage. + // + // +optional + // + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 88bc53a3ae53..99f5c0866f2f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -11,6 +11,7 @@ package v1alpha1 import ( appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -925,6 +926,11 @@ func (in *EnvoyProxyKubernetesProvider) DeepCopyInto(out *EnvoyProxyKubernetesPr *out = new(KubernetesServiceSpec) (*in).DeepCopyInto(*out) } + if in.EnvoyHpa != nil { + in, out := &in.EnvoyHpa, &out.EnvoyHpa + *out = new(KubernetesHorizontalPodAutoscalerSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyProxyKubernetesProvider. @@ -1428,6 +1434,38 @@ func (in *KubernetesDeploymentSpec) DeepCopy() *KubernetesDeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopyInto(out *KubernetesHorizontalPodAutoscalerSpec) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesHorizontalPodAutoscalerSpec. +func (in *KubernetesHorizontalPodAutoscalerSpec) DeepCopy() *KubernetesHorizontalPodAutoscalerSpec { + if in == nil { + return nil + } + out := new(KubernetesHorizontalPodAutoscalerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesPodSpec) DeepCopyInto(out *KubernetesPodSpec) { *out = *in diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 2b9373716345..65fabf07e02a 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5230,6 +5230,687 @@ spec: type: string type: object type: object + envoyHpa: + description: EnvoyHpa defines the Horizontal Pod Autoscaler + settings for Envoy Proxy Deployment. Once the HPA is being + set, Replicas field from EnvoyDeployment will be ignored. + properties: + behavior: + description: behavior configures the scaling behavior + of the target in both Up and Down directions (scaleUp + and scaleDown fields respectively). If not set, the + default HPAScalingRules for scale up and scale down + are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling + Down. If not set, the default value is to allow + to scale down to minReplicas pods, with a 300 second + stabilization window (i.e., the highest recommendation + for the last 300sec is used). + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling + Up. If not set, the default value is the higher + of: * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds No stabilization + is used.' + properties: + policies: + description: policies is a list of potential scaling + polices which can be used during scaling. At + least one policy must be specified, otherwise + the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: periodSeconds specifies the + window of time for which the policy should + hold true. PeriodSeconds must be greater + than zero and less than or equal to 1800 + (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: value contains the amount of + change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which + policy should be used. If not set, the default + value Max is used. + type: string + stabilizationWindowSeconds: + description: 'stabilizationWindowSeconds is the + number of seconds for which past recommendations + should be considered while scaling up or scaling + down. StabilizationWindowSeconds must be greater + than or equal to zero and less than or equal + to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object + maxReplicas: + default: 1 + description: maxReplicas is the upper limit for the number + of replicas to which the autoscaler can scale up. It + cannot be less that minReplicas. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + format: int32 + type: integer + metrics: + description: "metrics contains the specifications for + which to use to calculate the desired replica count + (the maximum replica count across all metrics will be + used). If left empty, it defaults to being based on + CPU utilization with average on 80% usage. \n See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior." + items: + description: MetricSpec specifies how to scale based + on a single metric (only `type` and one other matching + field should be set at once). + properties: + containerResource: + description: containerResource refers to a resource + metric (such as those specified in requests and + limits) known to Kubernetes describing a single + container in each pod of the current scale target + (e.g. CPU or memory). Such metrics are built in + to Kubernetes, and have special scaling options + on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature + and can be enabled by the HPAContainerMetrics + feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric + that is not associated with any Kubernetes object. + It allows autoscaling based on information coming + from components running outside of cluster (for + example length of queue in cloud messaging service, + or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing + a single kubernetes object (for example, hits-per-second + on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing + each pod in the current scale target (for example, + transactions-processed-per-second). The values + will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping. + When unset, just the metricName will be + used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric + (such as those specified in requests and limits) + known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special + scaling options on top of those available to normal + per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: averageUtilization is the target + value of the average of the resource metric + across all relevant pods, represented + as a percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It + should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to + a matching field in the object. Note: "ContainerResource" + type is available on when the feature-gate HPAContainerMetrics + is enabled' + type: string + required: + - type + type: object + type: array + minReplicas: + description: minReplicas is the lower limit for the number + of replicas to which the autoscaler can scale down. + It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec + format: int32 + type: integer + type: object + x-kubernetes-validations: + - message: maxReplicas must be greater than 0 + rule: self.maxReplicas > 0 + - message: maxReplicas cannot be less than minReplicas + rule: '!has(self.minReplicas) || self.maxReplicas >= self.minReplicas' envoyService: description: EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index f6b27d26702a..145f824e5a02 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,6 +26,7 @@ type ResourceRender interface { Service() (*corev1.Service, error) ConfigMap() (*corev1.ConfigMap, error) Deployment() (*appsv1.Deployment, error) + HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) } // Infra manages the creation and deletion of Kubernetes infrastructure @@ -68,6 +70,10 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to create or update service %s/%s", i.Namespace, r.Name()) } + if err := i.configureHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to configure hpa %s/%s", i.Namespace, r.Name()) + } + return nil } @@ -89,5 +95,9 @@ func (i *Infra) delete(ctx context.Context, r ResourceRender) error { return errors.Wrapf(err, "failed to delete service %s/%s", i.Namespace, r.Name()) } + if err := i.deleteHPA(ctx, r); err != nil { + return errors.Wrapf(err, "failed to delete hpa %s/%s", i.Namespace, r.Name()) + } + return nil } diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index af041ad33135..766482297cf3 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -10,11 +10,14 @@ import ( "reflect" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" ) // createOrUpdateServiceAccount creates a ServiceAccount in the kube api server based on the @@ -74,8 +77,52 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) Name: deployment.Name, } + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + var opts cmp.Options + if hpa != nil { + opts = append(opts, cmpopts.IgnoreFields(appsv1.DeploymentSpec{}, "Replicas")) + } + return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { - return !reflect.DeepEqual(deployment.Spec, current.Spec) + return !cmp.Equal(current.Spec, deployment.Spec, opts...) + }) +} + +// configureHPA configures HorizontalPodAutoscaler object in the kube api server based on the provided ResourceRender, +// there are two operations on this method: +// 1. if an HPA is defined and it doesn't exist then creates, or otherwise updates +// 2. if an HPA is not defined then delete the object irrespective of its existence +func (i *Infra) configureHPA(ctx context.Context, r ResourceRender) error { + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + // when HorizontalPodAutoscaler is disabled, + // then delete the object in the kube api server if any. + if hpa == nil { + hpa = &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.Client.Delete(ctx, hpa) + } + + current := &autoscalingv2.HorizontalPodAutoscaler{} + key := types.NamespacedName{ + Namespace: hpa.Namespace, + Name: hpa.Name, + } + + return i.Client.CreateOrUpdate(ctx, key, current, hpa, func() bool { + return !cmp.Equal(hpa.Spec, current.Spec) }) } @@ -145,3 +192,17 @@ func (i *Infra) deleteService(ctx context.Context, r ResourceRender) error { return i.Client.Delete(ctx, svc) } + +// deleteHpa deletes the Horizontal Pod Autoscaler associated to its renderer, if it exists. +func (i *Infra) deleteHPA(ctx context.Context, r ResourceRender) error { + hpa, err := r.HorizontalPodAutoscaler() + if err != nil { + return err + } + + if hpa == nil { + return nil + } + + return i.Client.Delete(ctx, hpa) +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index bc446f79f9e4..f463b1088b33 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -11,6 +11,7 @@ import ( "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -56,7 +57,7 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, }, nil @@ -110,7 +111,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, Annotations: annotations, }, @@ -135,7 +136,7 @@ func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: labels, }, Data: map[string]string{ @@ -192,7 +193,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: ExpectedResourceHashedName(r.infra.Name), + Name: r.Name(), Labels: dpLabels, }, Spec: appsv1.DeploymentSpec{ @@ -224,5 +225,46 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, } + // omit the deployment replicas if HPA is being set + if provider.GetEnvoyProxyKubeProvider().EnvoyHpa != nil { + deployment.Spec.Replicas = nil + } + return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + provider := r.infra.GetProxyConfig().GetEnvoyProxyProvider() + if provider.Type != egv1a1.ProviderTypeKubernetes { + return nil, fmt.Errorf("invalid provider type %v for Kubernetes infra manager", provider.Type) + } + + hpaConfig := provider.GetEnvoyProxyKubeProvider().EnvoyHpa + if hpaConfig == nil { + return nil, nil + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling/v2", + Kind: "HorizontalPodAutoscaler", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: r.Name(), + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: r.Name(), + }, + MinReplicas: hpaConfig.MinReplicas, + MaxReplicas: hpaConfig.MaxReplicas, + Metrics: hpaConfig.Metrics, + Behavior: hpaConfig.Behavior, + }, + } + + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 1c784927385f..6f5c33833af1 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -14,7 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/pointer" "sigs.k8s.io/yaml" @@ -23,6 +25,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/utils/ptr" ) const ( @@ -340,6 +343,7 @@ func TestDeployment(t *testing.T) { if tc.deploy != nil { kube.EnvoyDeployment = tc.deploy } + replace := egv1a1.BootstrapTypeReplace if tc.bootstrap != "" { tc.infra.Proxy.Config.Spec.Bootstrap = &egv1a1.ProxyBootstrap{ @@ -506,3 +510,83 @@ func loadServiceAccount() (*corev1.ServiceAccount, error) { _ = yaml.Unmarshal(saYAML, sa) return sa, nil } + +func TestHorizontalPodAutoscaler(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + infra *ir.Infra + hpa *egv1a1.KubernetesHorizontalPodAutoscalerSpec + }{ + { + caseName: "default", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{}, + }, + { + caseName: "custom", + infra: newTestInfra(), + hpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: 10, + Metrics: []autoscalingv2.MetricSpec{ + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](60), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + { + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](70), + }, + }, + Type: autoscalingv2.ResourceMetricSourceType, + }, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + provider := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() + provider.Kubernetes = egv1a1.DefaultEnvoyProxyKubeProvider() + + if tc.hpa != nil { + provider.Kubernetes.EnvoyHpa = tc.hpa + } + + provider.GetEnvoyProxyKubeProvider() + + r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra()) + hpa, err := r.HorizontalPodAutoscaler() + require.NoError(t, err) + + want, err := loadHPA(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, want, hpa) + }) + } +} + +func loadHPA(caseName string) (*autoscalingv2.HorizontalPodAutoscaler, error) { + hpaYAML, err := os.ReadFile(fmt.Sprintf("testdata/hpa/%s.yaml", caseName)) + if err != nil { + return nil, err + } + + hpa := &autoscalingv2.HorizontalPodAutoscaler{} + _ = yaml.Unmarshal(hpaYAML, hpa) + return hpa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml new file mode 100644 index 000000000000..17171f8abb3e --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/custom.yaml @@ -0,0 +1,25 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 60 + type: Utilization + type: Resource + - resource: + name: memory + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 5 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml new file mode 100644 index 000000000000..d11b7e476369 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/hpa/default.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + metrics: + - resource: + name: cpu + target: + averageUtilization: 80 + type: Utilization + type: Resource + maxReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: envoy-default-37a8eec1 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index f39961795ccc..bb7f2ee598d1 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -7,6 +7,7 @@ package ratelimit import ( appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -202,3 +203,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return deployment, nil } + +func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { + return nil, nil +} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 7379bf3cef7d..975e60f8770b 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -647,6 +647,7 @@ _Appears in:_ | --- | --- | | `envoyDeployment` _[KubernetesDeploymentSpec](#kubernetesdeploymentspec)_ | EnvoyDeployment defines the desired state of the Envoy deployment resource. If unspecified, default settings for the managed Envoy deployment resource are applied. | | `envoyService` _[KubernetesServiceSpec](#kubernetesservicespec)_ | EnvoyService defines the desired state of the Envoy service resource. If unspecified, default settings for the managed Envoy service resource are applied. | +| `envoyHpa` _[KubernetesHorizontalPodAutoscalerSpec](#kuberneteshorizontalpodautoscalerspec)_ | EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment. Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. | #### EnvoyProxyProvider @@ -976,6 +977,24 @@ _Appears in:_ | `initContainers` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#container-v1-core) array_ | List of initialization containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +#### KubernetesHorizontalPodAutoscalerSpec + + + +KubernetesHorizontalPodAutoscalerSpec defines Kubernetes Horizontal Pod Autoscaler settings of Envoy Proxy Deployment + +_Appears in:_ +- [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider) + +| Field | Description | +| --- | --- | +| `minReplicas` _integer_ | minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 replica. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | +| `maxReplicas` _integer_ | maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec | +| `metrics` _[MetricSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#metricspec-v2-autoscaling) array_ | metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). + See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | +| `behavior` _[HorizontalPodAutoscalerBehavior](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#horizontalpodautoscalerbehavior-v2-autoscaling)_ | behavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default HPAScalingRules for scale up and scale down are used. See k8s.io.autoscaling.v2.HorizontalPodAutoScalerBehavior. | + + #### KubernetesPodSpec