feat: expose envoyproxy image by proxyConfig

Signed-off-by: bitliu <[email protected]>

api/config/v1alpha1/helpers.go

@@ -9,6 +9,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 )
 
 // DefaultEnvoyGateway returns a new EnvoyGateway with default configuration parameters.
@@ -106,6 +107,11 @@ func DefaultKubernetesDeploymentReplicas() *int32 {
 	return &repl
 }
 
+// DefaultKubernetesContainerImage returns the default envoyproxy image.
+func DefaultKubernetesContainerImage() *string {
+	return pointer.String(DefaultEnvoyProxyImage)
+}
+
 // DefaultKubernetesDeployment returns a new KubernetesDeploymentSpec with default settings.
 func DefaultKubernetesDeployment() *KubernetesDeploymentSpec {
 	return &KubernetesDeploymentSpec{
@@ -124,6 +130,7 @@ func DefaultKubernetesPod() *KubernetesPodSpec {
 func DefaultKubernetesContainer() *KubernetesContainerSpec {
 	return &KubernetesContainerSpec{
 		Resources: DefaultResourceRequirements(),
+		Image:     DefaultKubernetesContainerImage(),
 	}
 }
 
@@ -187,6 +194,10 @@ func (r *EnvoyProxyProvider) GetEnvoyProxyKubeProvider() *EnvoyProxyKubernetesPr
 		r.Kubernetes.EnvoyDeployment.Container.Resources = DefaultResourceRequirements()
 	}
 
+	if r.Kubernetes.EnvoyDeployment.Container.Image == nil {
+		r.Kubernetes.EnvoyDeployment.Container.Image = DefaultKubernetesContainerImage()
+	}
+
 	if r.Kubernetes.EnvoyService == nil {
 		r.Kubernetes.EnvoyService = DefaultKubernetesService()
 	}
@@ -231,5 +242,9 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern
 		r.Kubernetes.RateLimitDeployment.Container.Resources = DefaultResourceRequirements()
 	}
 
+	if r.Kubernetes.RateLimitDeployment.Container.Image == nil {
+		r.Kubernetes.RateLimitDeployment.Container.Image = DefaultKubernetesContainerImage()
+	}
+
 	return r.Kubernetes
 }

api/config/v1alpha1/shared_types.go

@@ -14,6 +14,8 @@ const (
 	DefaultDeploymentCPUResourceRequests = "100m"
 	// DefaultDeploymentMemoryResourceRequests for deployment memory resource
 	DefaultDeploymentMemoryResourceRequests = "512Mi"
+	// DefaultEnvoyProxyImage is the default image used by envoyproxy
+	DefaultEnvoyProxyImage = "envoyproxy/envoy-dev:latest"
 )
 
 // GroupVersionKind unambiguously identifies a Kind.
@@ -87,6 +89,11 @@ type KubernetesContainerSpec struct {
 	//
 	// +optional
 	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
+
+	// Image specifies the EnvoyProxy container image to be used, instead of the default image.
+	//
+	// +optional
+	Image *string `json:"image,omitempty"`
 }
 
 // ServiceType string describes ingress methods for a service

api/config/v1alpha1/validate_test.go

@@ -241,10 +241,10 @@ func TestEnvoyGatewayProvider(t *testing.T) {
 
 	envoyGatewayProvider := envoyGateway.GetEnvoyGatewayProvider()
 	assert.True(t, envoyGatewayProvider.Kubernetes == nil)
-	assert.True(t, reflect.DeepEqual(envoyGateway.Provider, envoyGatewayProvider))
+	assert.Equal(t, envoyGateway.Provider, envoyGatewayProvider)
 
 	envoyGatewayProvider.Kubernetes = DefaultEnvoyGatewayKubeProvider()
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment, DefaultKubernetesDeployment()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment, DefaultKubernetesDeployment())
 
 	envoyGatewayProvider.Kubernetes = &EnvoyGatewayKubernetesProvider{}
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment == nil)
@@ -265,24 +265,27 @@ func TestEnvoyGatewayProvider(t *testing.T) {
 		Container: &KubernetesContainerSpec{
 			Resources:       nil,
 			SecurityContext: nil,
+			Image:           nil,
 		},
 	}}
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Resources == nil)
 	envoyGatewayProvider.GetEnvoyGatewayKubeProvider()
 
 	assert.True(t, envoyGatewayProvider.Kubernetes != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes, envoyGatewayKubeProvider))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes, envoyGatewayKubeProvider)
 
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment, DefaultKubernetesDeployment()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment, DefaultKubernetesDeployment())
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Replicas != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment.Replicas, DefaultKubernetesDeploymentReplicas()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Replicas, DefaultKubernetesDeploymentReplicas())
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Pod != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment.Pod, DefaultKubernetesPod()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Pod, DefaultKubernetesPod())
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container, DefaultKubernetesContainer()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container, DefaultKubernetesContainer())
 	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Resources != nil)
-	assert.True(t, reflect.DeepEqual(envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Resources, DefaultResourceRequirements()))
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Resources, DefaultResourceRequirements())
+	assert.True(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Image != nil)
+	assert.Equal(t, envoyGatewayProvider.Kubernetes.RateLimitDeployment.Container.Image, DefaultKubernetesContainerImage())
 }
 
 func TestEnvoyProxyProvider(t *testing.T) {

charts/gateway-helm/crds/generated/config.gateway.envoyproxy.io_envoyproxies.yaml

@@ -96,6 +96,10 @@ spec:
                             description: Container defines the resources and securityContext
                               of container.
                             properties:
+                              image:
+                                description: Image specifies the EnvoyProxy container
+                                  image to be used, instead of the default image.
+                                type: string
                               resources:
                                 description: 'Resources required by this container.
                                   More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'

docs/latest/api/config_types.md

@@ -257,6 +257,7 @@ _Appears in:_
 | --- | --- |
 | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#resourcerequirements-v1-core)_ | Resources required by this container. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
 | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#securitycontext-v1-core)_ | SecurityContext defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ |
+| `image` _string_ | Image specifies the EnvoyProxy container image to be used, instead of the default image. |
 
 
 ## KubernetesDeploymentSpec

docs/latest/design/system-design.md

@@ -39,7 +39,7 @@ defined as Kubernetes resources that provide the following services:
 * Infrastructure Management- Manage the data plane infrastructure, i.e. deploy, upgrade, etc. This configuration is
   expressed through [GatewayClass][gc] and [Gateway][gw] resources. The `EnvoyProxy` [Custom Resource][cr] can be
   referenced by `gatewayclass.spec.parametersRef` to modify data plane infrastructure default parameters,
-  e.g. expose Envoy network endpoints using a NodePort service instead of a LoadBalancer service.
+  e.g. expose Envoy network endpoints using a `ClusterIP` service instead of a `LoadBalancer` service.
 * Traffic Routing- Define how to handle application-level requests to backend services. For example, route all HTTP
   requests for "www.example.com" to a backend service running a web server. This configuration is expressed through
   [HTTPRoute][hroute] and [TLSRoute][troute] resources that match, filter, and route traffic to a [backend][be].

docs/latest/user/customize-envoyproxy.md

@@ -0,0 +1,183 @@
+# Customize EnvoyProxy
+
+Envoy Gateway provides a [EnvoyProxy][] CRD that can be linked to the ParametersRef
+in GatewayClass y cluster admins to customize the managed EnvoyProxy Deployment and
+Service. To learn more about GatewayClass and ParametersRef, please refer to [Gateway API documentation][].
+
+## Installation
+
+Follow the steps from the [Quickstart Guide](quickstart.md) to install Envoy Gateway and the example manifest.
+Before proceeding, you should be able to query the example backend using HTTP.
+
+## Add GatewayClass ParametersRef
+
+First, you need to add ParametersRef in GatewayClass, and refer to EnvoyProxy Config:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: GatewayClass
+metadata:
+  name: eg
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: config.gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: custom-proxy-config
+    namespace: envoy-gateway-system
+EOF
+```
+
+## Customize EnvoyProxy Deployment Replicas
+
+You can customize the EnvoyProxy Deployment Replicas via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        replicas: 2
+EOF
+```
+
+After you apply the config, you should see the replicas of envoyproxy changes to 2.
+And also you can dynamically change the value.
+
+``` shell
+kubectl get deployment envoy-gateway
+```
+
+## Customize EnvoyProxy Image
+
+You can customize the EnvoyProxy Image via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        container:
+          image: envoyproxy/envoy:v1.25-latest
+EOF
+```
+
+After applying the config, you can get the deployment image, and see it has changed.
+
+## Customize EnvoyProxy Pod Annotations
+
+You can customize the EnvoyProxy Pod Annotations via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        pod:
+          annotations:
+            custom1: deploy-annotation1
+            custom2: deploy-annotation2
+EOF
+```
+
+After applying the config, you can get the envoyproxy pods, and see new annotations has been added.
+
+## Customize EnvoyProxy Deployment Resources
+
+You can customize the EnvoyProxy Deployment Resources via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        container:
+          resources:
+            requests:
+              cpu: 150m
+              memory: 640Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
+EOF
+```
+
+After applying the config, you can get the envoyproxy deployment, and see resources has been changed.
+
+## Customize EnvoyProxy Service Annotations
+
+You can customize the EnvoyProxy Service Annotations via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyService:
+        annotations:
+          custom1: svc-annotation1
+          custom2: svc-annotation2
+
+EOF
+```
+
+After applying the config, you can get the envoyproxy service, and see annotations has been added.
+
+## Customize EnvoyProxy Bootstrap Config
+
+You can customize the EnvoyProxy Bootstrap Config via EnvoyProxy Config like:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: envoy-gateway-system
+spec:
+  bootstrap: |
+    xxxxxxxxxx
+
+EOF
+```
+
+After applying the config, the bootstrap config will be overridden by the new config you provided.
+Envoy Gateway will use Webhook to validate the bootstrap config you provided.
+
+[Gateway API documentation]: https://gateway-api.sigs.k8s.io/
+[EnvoyProxy]: https://www.envoyproxy.io/

docs/latest/user/grpc-routing.md

@@ -138,7 +138,6 @@ Test GRPC routing to the `yages` backend using the [grpcurl][] command.
 grpcurl -plaintext -authority=grpc-example.com ${GATEWAY_HOST}:80 yages.Echo/Ping
 ```
 
-
 [GRPCRoute]: https://gateway-api.sigs.k8s.io/api-types/grpcroute/
 [Gateway API documentation]: https://gateway-api.sigs.k8s.io/
 [GatewayClass]: https://gateway-api.sigs.k8s.io/api-types/gatewayclass/

docs/latest/user_docs.rst

@@ -21,3 +21,4 @@ Learn how to deploy, use, and operate Envoy Gateway.
   user/authn
   user/rate-limit
   user/egctl
+  user/customize-envoyproxy

examples/kubernetes/envoy-proxy-config.yaml

@@ -0,0 +1,38 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: GatewayClass
+metadata:
+  name: eg
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: config.gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: config
+    namespace: envoy-gateway-system
+---
+apiVersion: config.gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        replicas: 2
+        image: envoyproxy/envoy:v1.25-latest
+        podAnnotations:
+          custom1: deploy-annotation1
+          custom2: deploy-annotation2
+        resources:
+          requests:
+            cpu: 150m
+            memory: 640Mi
+          limits:
+            cpu: 500m
+            memory: 1Gi
+      envoyService:
+        annotations:
+          custom1: svc-annotation1
+          custom2: svc-annotation2