feat: allow running EnvoyProxy as DaemonSet

api/v1alpha1/envoygateway_helpers.go

@@ -228,11 +228,20 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern
 		r.Kubernetes.LeaderElection = DefaultLeaderElection()
 	}
 
-	if r.Kubernetes.RateLimitDeployment == nil {
+	// if RateLimitDeployment and RateLimitDaemonset are both nil, use RateLimitDeployment
+	if r.Kubernetes.RateLimitDeployment == nil && r.Kubernetes.RateLimitDaemonset == nil {
 		r.Kubernetes.RateLimitDeployment = DefaultKubernetesDeployment(DefaultRateLimitImage)
 	}
 
-	r.Kubernetes.RateLimitDeployment.defaultKubernetesDeploymentSpec(DefaultRateLimitImage)
+	// if use RateLimitDeployment, set default values
+	if r.Kubernetes.RateLimitDeployment != nil {
+		r.Kubernetes.RateLimitDeployment.defaultKubernetesDeploymentSpec(DefaultRateLimitImage)
+	}
+
+	// if use RateLimitDaemonset, set default values
+	if r.Kubernetes.RateLimitDaemonset != nil {
+		r.Kubernetes.RateLimitDaemonset.defaultKubernetesDaemonSetSpec(DefaultRateLimitImage)
+	}
 
 	if r.Kubernetes.ShutdownManager == nil {
 		r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}

api/v1alpha1/envoygateway_types.go

@@ -202,6 +202,13 @@ type EnvoyGatewayKubernetesProvider struct {
 	// +optional
 	RateLimitDeployment *KubernetesDeploymentSpec `json:"rateLimitDeployment,omitempty"`
 
+	// RateLimitDaemonset defines the desired state of the Envoy ratelimit daemonset resource.
+	// If unspecified, default settings for the managed Envoy ratelimit daemonset resource
+	// are applied.
+	//
+	// +optional
+	RateLimitDaemonset *KubernetesDaemonSetSpec `json:"rateLimitDaemonset,omitempty"`
+
 	// Watch holds configuration of which input resources should be watched and reconciled.
 	// +optional
 	Watch *KubernetesWatchMode `json:"watch,omitempty"`

charts/gateway-helm/templates/_rbac.tpl

@@ -43,6 +43,7 @@ apiGroups:
 - apps
 resources:
 - deployments
+- daemonsets
 verbs:
 - get
 - list

internal/gatewayapi/status/gateway.go

@@ -13,6 +13,7 @@
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
@@ -30,8 +31,8 @@
 
 // UpdateGatewayStatusProgrammedCondition updates the status addresses for the provided gateway
 // based on the status IP/Hostname of svc and updates the Programmed condition based on the
-// service and deployment state.
-func UpdateGatewayStatusProgrammedCondition(gw *gwapiv1.Gateway, svc *corev1.Service, deployment *appsv1.Deployment, nodeAddresses ...string) {
+// service and deployment or daemonset state.
+func UpdateGatewayStatusProgrammedCondition(gw *gwapiv1.Gateway, svc *corev1.Service, envoyObj client.Object, nodeAddresses ...string) {
 	var addresses, hostnames []string
 	// Update the status addresses field.
 	if svc != nil {
@@ -98,7 +99,7 @@
 	}
 
 	// Update the programmed condition.
-	updateGatewayProgrammedCondition(gw, deployment)
+	updateGatewayProgrammedCondition(gw, envoyObj)
 }
 
 func SetGatewayListenerStatusCondition(gateway *gwapiv1.Gateway, listenerStatusIdx int,
@@ -132,13 +133,13 @@
 const (
 	messageAddressNotAssigned  = "No addresses have been assigned to the Gateway"
 	messageFmtTooManyAddresses = "Too many addresses (%d) have been assigned to the Gateway, the maximum number of addresses is 16"
-	messageNoResources         = "Deployment replicas unavailable"
+	messageNoResources         = "Envoy replicas unavailable"
 	messageFmtProgrammed       = "Address assigned to the Gateway, %d/%d envoy Deployment replicas available"
 )
 
 // updateGatewayProgrammedCondition computes the Gateway Programmed status condition.
-// Programmed condition surfaces true when the Envoy Deployment status is ready.
-func updateGatewayProgrammedCondition(gw *gwapiv1.Gateway, deployment *appsv1.Deployment) {
+// Programmed condition surfaces true when the Envoy Deployment or Daemonset status is ready.
+func updateGatewayProgrammedCondition(gw *gwapiv1.Gateway, envoyObj client.Object) {
 	if len(gw.Status.Addresses) == 0 {
 		gw.Status.Conditions = MergeConditions(gw.Status.Conditions,
 			newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionFalse, string(gwapiv1.GatewayReasonAddressNotAssigned),
@@ -157,17 +158,27 @@
 		return
 	}
 
-	// If there are no available replicas for the Envoy Deployment, don't
-	// mark the Gateway as ready yet.
-
-	if deployment == nil || deployment.Status.AvailableReplicas == 0 {
-		gw.Status.Conditions = MergeConditions(gw.Status.Conditions,
-			newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionFalse, string(gwapiv1.GatewayReasonNoResources),
-				messageNoResources, time.Now(), gw.Generation))
-		return
+	// Check for available Envoy replicas and if found mark the gateway as ready.
+	switch obj := envoyObj.(type) {
+	case *appsv1.Deployment:
+		if obj != nil && obj.Status.AvailableReplicas > 0 {
+			gw.Status.Conditions = MergeConditions(gw.Status.Conditions,
+				newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionTrue, string(gwapiv1.GatewayConditionProgrammed),
+					fmt.Sprintf(messageFmtProgrammed, obj.Status.AvailableReplicas, obj.Status.Replicas), time.Now(), gw.Generation))
+			return
+		}
+	case *appsv1.DaemonSet:
+		if obj != nil && obj.Status.NumberAvailable > 0 {
+			gw.Status.Conditions = MergeConditions(gw.Status.Conditions,
+				newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionTrue, string(gwapiv1.GatewayConditionProgrammed),
+					fmt.Sprintf(messageFmtProgrammed, obj.Status.NumberAvailable, obj.Status.CurrentNumberScheduled), time.Now(), gw.Generation))
+			return
+		}
 	}
 
+	// If there are no available replicas for the Envoy Deployment or
+	// Envoy Daemonset, don't mark the Gateway as ready yet.
 	gw.Status.Conditions = MergeConditions(gw.Status.Conditions,
-		newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionTrue, string(gwapiv1.GatewayConditionProgrammed),
-			fmt.Sprintf(messageFmtProgrammed, deployment.Status.AvailableReplicas, deployment.Status.Replicas), time.Now(), gw.Generation))
+		newCondition(string(gwapiv1.GatewayConditionProgrammed), metav1.ConditionFalse, string(gwapiv1.GatewayReasonNoResources),
+			messageNoResources, time.Now(), gw.Generation))
 }

internal/infrastructure/kubernetes/ratelimit/resource.go

@@ -138,7 +138,7 @@ func rateLimitLabels() map[string]string {
 }
 
 // expectedRateLimitContainers returns expected rateLimit containers.
-func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec,
+func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitContainerSpec *egv1a1.KubernetesContainerSpec,
 	namespace string,
 ) []corev1.Container {
 	ports := []corev1.ContainerPort{
@@ -152,16 +152,16 @@ func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeploymen
 	containers := []corev1.Container{
 		{
 			Name:            InfraName,
-			Image:           *rateLimitDeployment.Container.Image,
+			Image:           *rateLimitContainerSpec.Image,
 			ImagePullPolicy: corev1.PullIfNotPresent,
 			Command: []string{
 				"/bin/ratelimit",
 			},
-			Env:                      expectedRateLimitContainerEnv(rateLimit, rateLimitDeployment, namespace),
+			Env:                      expectedRateLimitContainerEnv(rateLimit, rateLimitContainerSpec, namespace),
 			Ports:                    ports,
-			Resources:                *rateLimitDeployment.Container.Resources,
-			SecurityContext:          expectedRateLimitContainerSecurityContext(rateLimitDeployment),
-			VolumeMounts:             expectedContainerVolumeMounts(rateLimit, rateLimitDeployment),
+			Resources:                *rateLimitContainerSpec.Resources,
+			SecurityContext:          expectedRateLimitContainerSecurityContext(rateLimitContainerSpec),
+			VolumeMounts:             expectedContainerVolumeMounts(rateLimit, rateLimitContainerSpec),
 			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
 			TerminationMessagePath:   "/dev/termination-log",
 			StartupProbe: &corev1.Probe{
@@ -197,7 +197,7 @@ func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeploymen
 }
 
 // expectedContainerVolumeMounts returns expected rateLimit container volume mounts.
-func expectedContainerVolumeMounts(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) []corev1.VolumeMount {
+func expectedContainerVolumeMounts(rateLimit *egv1a1.RateLimit, rateLimitContainerSpec *egv1a1.KubernetesContainerSpec) []corev1.VolumeMount {
 	var volumeMounts []corev1.VolumeMount
 
 	// mount the cert
@@ -223,11 +223,11 @@ func expectedContainerVolumeMounts(rateLimit *egv1a1.RateLimit, rateLimitDeploym
 		})
 	}
 
-	return resource.ExpectedContainerVolumeMounts(rateLimitDeployment.Container, volumeMounts)
+	return resource.ExpectedContainerVolumeMounts(rateLimitContainerSpec, volumeMounts)
 }
 
 // expectedDeploymentVolumes returns expected rateLimit deployment volumes.
-func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) []corev1.Volume {
+func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitPodSpec *egv1a1.KubernetesPodSpec) []corev1.Volume {
 	var volumes []corev1.Volume
 
 	if rateLimit.Backend.Redis != nil &&
@@ -269,11 +269,11 @@ func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitDeployment
 		})
 	}
 
-	return resource.ExpectedVolumes(rateLimitDeployment.Pod, volumes)
+	return resource.ExpectedVolumes(rateLimitPodSpec, volumes)
 }
 
 // expectedRateLimitContainerEnv returns expected rateLimit container envs.
-func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec,
+func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitContainerSpec *egv1a1.KubernetesContainerSpec,
 	namespace string,
 ) []corev1.EnvVar {
 	env := []corev1.EnvVar{
@@ -445,7 +445,7 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym
 		env = append(env, tracingEnvs...)
 	}
 
-	return resource.ExpectedContainerEnv(rateLimitDeployment.Container, env)
+	return resource.ExpectedContainerEnv(rateLimitContainerSpec, env)
 }
 
 // Validate the ratelimit tls secret validating.
@@ -489,9 +489,9 @@ func checkTraceEndpointScheme(url string) string {
 	return fmt.Sprintf("%s%s", httpScheme, url)
 }
 
-func expectedRateLimitContainerSecurityContext(rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) *corev1.SecurityContext {
-	if rateLimitDeployment.Container.SecurityContext != nil {
-		return rateLimitDeployment.Container.SecurityContext
+func expectedRateLimitContainerSecurityContext(rateLimitContainerSpec *egv1a1.KubernetesContainerSpec) *corev1.SecurityContext {
+	if rateLimitContainerSpec.SecurityContext != nil {
+		return rateLimitContainerSpec.SecurityContext
 	}
 	return defaultSecurityContext()
 }

internal/infrastructure/kubernetes/ratelimit/resource_provider.go

@@ -27,6 +27,7 @@
 // but also the key for the uid of their ownerReference.
 const (
 	ResourceKindService        = "Service"
+	ResourceKindDaemonset      = "Daemonset"
 	ResourceKindDeployment     = "Deployment"
 	ResourceKindServiceAccount = "ServiceAccount"
 	appsAPIVersion             = "apps/v1"
@@ -41,6 +42,7 @@
 
 	rateLimit           *egv1a1.RateLimit
 	rateLimitDeployment *egv1a1.KubernetesDeploymentSpec
+	rateLimitDaemonset  *egv1a1.KubernetesDaemonSetSpec
 
 	// ownerReferenceUID store the uid of its owner reference.
 	ownerReferenceUID map[string]types.UID
@@ -51,6 +53,7 @@
 	return &ResourceRender{
 		Namespace:           ns,
 		rateLimit:           gateway.RateLimit,
+		rateLimitDaemonset:  gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDaemonset,
 		rateLimitDeployment: gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment,
 		ownerReferenceUID:   ownerReferenceUID,
 	}
@@ -196,7 +199,7 @@
 		return nil, er
 	}
 
-	containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment, r.Namespace)
+	containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment.Container, r.Namespace)
 	selector := resource.GetSelector(rateLimitLabels())
 
 	podLabels := rateLimitLabels()
@@ -250,7 +253,7 @@
 					RestartPolicy:                 corev1.RestartPolicyAlways,
 					SchedulerName:                 "default-scheduler",
 					SecurityContext:               r.rateLimitDeployment.Pod.SecurityContext,
-					Volumes:                       expectedDeploymentVolumes(r.rateLimit, r.rateLimitDeployment),
+					Volumes:                       expectedDeploymentVolumes(r.rateLimit, r.rateLimitDeployment.Pod),
 					Affinity:                      r.rateLimitDeployment.Pod.Affinity,
 					Tolerations:                   r.rateLimitDeployment.Pod.Tolerations,
 					ImagePullSecrets:              r.rateLimitDeployment.Pod.ImagePullSecrets,
@@ -294,12 +297,105 @@
 
 // DaemonSetSpec returns the `DaemonSet` sets spec.
 func (r *ResourceRender) DaemonSetSpec() (*egv1a1.KubernetesDaemonSetSpec, error) {
-	return nil, nil
+	return r.rateLimitDaemonset, nil
 }
 
-// TODO: implement this method
 func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) {
-	return nil, nil
+	// If daemonset config is nil,ignore Daemonset.
+	if daemonsetConfig, er := r.DaemonSetSpec(); daemonsetConfig == nil {
+		return nil, er
+	}
+
+	containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDaemonset.Container, r.Namespace)
+	selector := resource.GetSelector(rateLimitLabels())
+
+	podLabels := rateLimitLabels()
+	if r.rateLimitDaemonset.Pod.Labels != nil {
+		maps.Copy(podLabels, r.rateLimitDaemonset.Pod.Labels)
+		// Copy overwrites values in the dest map if they exist in the src map https://pkg.go.dev/maps#Copy
+		// It's applied again with the rateLimitLabels that are used as deployment selector to ensure those are not overwritten by user input
+		maps.Copy(podLabels, rateLimitLabels())
+	}
+
+	var podAnnotations map[string]string
+	if enablePrometheus(r.rateLimit) {
+		podAnnotations = map[string]string{
+			"prometheus.io/path":   "/metrics",
+			"prometheus.io/port":   strconv.Itoa(PrometheusPort),
+			"prometheus.io/scrape": "true",
+		}
+	}
+	if r.rateLimitDaemonset.Pod.Annotations != nil {
+		if podAnnotations != nil {
+			maps.Copy(podAnnotations, r.rateLimitDaemonset.Pod.Annotations)
+		} else {
+			podAnnotations = r.rateLimitDaemonset.Pod.Annotations
+		}
+	}
+
+	daemonset := &appsv1.DaemonSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       ResourceKindDaemonset,
+			APIVersion: appsAPIVersion,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.Namespace,
+			Labels:    rateLimitLabels(),
+		},
+		Spec: appsv1.DaemonSetSpec{
+			UpdateStrategy: *r.rateLimitDaemonset.Strategy,
+			Selector:       selector,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      podLabels,
+					Annotations: podAnnotations,
+				},
+				Spec: corev1.PodSpec{
+					Containers:                    containers,
+					ServiceAccountName:            InfraName,
+					AutomountServiceAccountToken:  ptr.To(false),
+					TerminationGracePeriodSeconds: ptr.To[int64](300),
+					DNSPolicy:                     corev1.DNSClusterFirst,
+					RestartPolicy:                 corev1.RestartPolicyAlways,
+					SchedulerName:                 "default-scheduler",
+					SecurityContext:               r.rateLimitDaemonset.Pod.SecurityContext,
+					Volumes:                       expectedDeploymentVolumes(r.rateLimit, r.rateLimitDaemonset.Pod),
+					Affinity:                      r.rateLimitDaemonset.Pod.Affinity,
+					Tolerations:                   r.rateLimitDaemonset.Pod.Tolerations,
+					ImagePullSecrets:              r.rateLimitDaemonset.Pod.ImagePullSecrets,
+					NodeSelector:                  r.rateLimitDaemonset.Pod.NodeSelector,
+				},
+			},
+		},
+	}
+
+	// set name
+	if r.rateLimitDaemonset.Name != nil {
+		daemonset.ObjectMeta.Name = *r.rateLimitDaemonset.Name
+	} else {
+		daemonset.ObjectMeta.Name = r.Name()
+	}
+
+	if r.ownerReferenceUID != nil {
+		if uid, ok := r.ownerReferenceUID[ResourceKindDaemonset]; ok {
+			daemonset.OwnerReferences = []metav1.OwnerReference{
+				{
+					Kind:       ResourceKindDaemonset,
+					APIVersion: appsAPIVersion,
+					Name:       "envoy-gateway",
+					UID:        uid,
+				},
+			}
+		}
+	}
+
+	// apply merge patch to deployment
+	var err error
+	if daemonset, err = r.rateLimitDaemonset.ApplyMergePatch(daemonset); err != nil {
+		return nil, err
+	}
+
+	return daemonset, nil
 }
 
 // HorizontalPodAutoscalerSpec returns the `HorizontalPodAutoscaler` sets spec.