From 02b9e68540a214fc5f6311e9821edf46d1fd612a Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Tue, 11 Apr 2023 09:42:05 +0800 Subject: [PATCH 1/3] refactor kubernetes Signed-off-by: hejianpeng --- internal/cmd/egctl/translate.go | 4 +- .../kubernetes/applier/apply.go | 187 ++++++++++++ .../infrastructure/kubernetes/configmap.go | 79 ----- .../infrastructure/kubernetes/deployment.go | 55 ---- .../kubernetes/deployment_test.go | 134 --------- internal/infrastructure/kubernetes/infra.go | 159 ++++++++++- internal/infrastructure/kubernetes/labels.go | 18 -- .../resource_provider.go} | 177 +++++++++--- .../proxy/resource_provider_test.go | 251 ++++++++++++++++ .../kubernetes/proxy/testdata/configmap.yaml | 12 + .../proxy/testdata/deployments/bootstrap.yaml | 84 ++++++ .../proxy/testdata/deployments/custom.yaml | 160 +++++++++++ .../proxy/testdata/deployments/default.yaml | 151 ++++++++++ .../proxy/testdata/serviceaccount.yaml | 9 + .../proxy/testdata/services/custom.yaml | 27 ++ .../proxy/testdata/services/default.yaml | 26 ++ .../kubernetes/proxy_configmap.go | 61 ---- .../kubernetes/proxy_configmap_test.go | 61 ++-- .../kubernetes/proxy_deployment_test.go | 268 ++--------------- .../infrastructure/kubernetes/proxy_infra.go | 39 +-- .../kubernetes/proxy_infra_test.go | 48 +++- .../infrastructure/kubernetes/proxy_labels.go | 23 -- .../kubernetes/proxy_labels_test.go | 37 --- .../kubernetes/proxy_service.go | 90 ------ .../kubernetes/proxy_service_test.go | 129 +-------- .../kubernetes/proxy_serviceaccount.go | 61 ---- .../kubernetes/proxy_serviceaccount_test.go | 62 ++-- .../kubernetes/ratelimit/deployment.go | 168 +++++++++++ .../kubernetes/ratelimit/resource_provider.go | 116 ++++++++ .../ratelimit/resource_provider_test.go | 269 ++++++++++++++++++ .../testdata/deployments/custom.yaml | 75 +++++ .../testdata/deployments/default.yaml | 66 +++++ .../testdata/envoy-ratelimit-configmap.yaml | 22 ++ .../testdata/envoy-ratelimit-service.yaml | 18 ++ .../envoy-ratelimit-serviceaccount.yaml | 5 + .../kubernetes/ratelimit/utils.go | 14 + .../kubernetes/ratelimit_configmap.go | 53 ---- .../kubernetes/ratelimit_configmap_test.go | 85 +++--- .../kubernetes/ratelimit_deployment.go | 193 ------------- .../kubernetes/ratelimit_deployment_test.go | 198 +++---------- .../kubernetes/ratelimit_infra.go | 38 +-- .../kubernetes/ratelimit_infra_test.go | 122 ++++++++ .../kubernetes/ratelimit_labels.go | 13 - .../kubernetes/ratelimit_labels_test.go | 34 --- .../kubernetes/ratelimit_service.go | 71 ----- .../kubernetes/ratelimit_service_test.go | 47 +-- .../kubernetes/ratelimit_serviceaccount.go | 49 ---- .../ratelimit_serviceaccount_test.go | 74 ++--- internal/infrastructure/kubernetes/service.go | 70 ----- .../infrastructure/kubernetes/service_test.go | 117 -------- .../kubernetes/serviceaccount.go | 55 ---- .../infrastructure/kubernetes/utils/utils.go | 83 ++++++ .../kubernetes/utils/utils_test.go | 75 +++++ internal/xds/translator/runner/runner.go | 4 +- internal/xds/translator/translator_test.go | 6 +- 55 files changed, 2477 insertions(+), 2075 deletions(-) create mode 100644 internal/infrastructure/kubernetes/applier/apply.go delete mode 100644 internal/infrastructure/kubernetes/configmap.go delete mode 100644 internal/infrastructure/kubernetes/deployment.go delete mode 100644 internal/infrastructure/kubernetes/deployment_test.go delete mode 100644 internal/infrastructure/kubernetes/labels.go rename internal/infrastructure/kubernetes/{proxy_deployment.go => proxy/resource_provider.go} (56%) create mode 100644 internal/infrastructure/kubernetes/proxy/resource_provider_test.go create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/configmap.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/serviceaccount.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/services/custom.yaml create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/services/default.yaml delete mode 100644 internal/infrastructure/kubernetes/proxy_configmap.go delete mode 100644 internal/infrastructure/kubernetes/proxy_labels.go delete mode 100644 internal/infrastructure/kubernetes/proxy_labels_test.go delete mode 100644 internal/infrastructure/kubernetes/proxy_service.go delete mode 100644 internal/infrastructure/kubernetes/proxy_serviceaccount.go create mode 100644 internal/infrastructure/kubernetes/ratelimit/deployment.go create mode 100644 internal/infrastructure/kubernetes/ratelimit/resource_provider.go create mode 100644 internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-configmap.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-service.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-serviceaccount.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/utils.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_configmap.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_deployment.go create mode 100644 internal/infrastructure/kubernetes/ratelimit_infra_test.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_labels.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_labels_test.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_service.go delete mode 100644 internal/infrastructure/kubernetes/ratelimit_serviceaccount.go delete mode 100644 internal/infrastructure/kubernetes/service.go delete mode 100644 internal/infrastructure/kubernetes/service_test.go delete mode 100644 internal/infrastructure/kubernetes/serviceaccount.go create mode 100644 internal/infrastructure/kubernetes/utils/utils.go create mode 100644 internal/infrastructure/kubernetes/utils/utils_test.go diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go index ab629577ca5..59946138881 100644 --- a/internal/cmd/egctl/translate.go +++ b/internal/cmd/egctl/translate.go @@ -34,7 +34,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" - infra "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/status" "github.com/envoyproxy/gateway/internal/xds/bootstrap" "github.com/envoyproxy/gateway/internal/xds/translator" @@ -297,7 +297,7 @@ func translateGatewayAPIToXds(resourceType string, resources *gatewayapi.Resourc xTranslator := &translator.Translator{ // Set some default settings for translation GlobalRateLimit: &translator.GlobalRateLimitSettings{ - ServiceURL: infra.GetRateLimitServiceURL("envoy-gateway"), + ServiceURL: ratelimit.GetServiceURL("envoy-gateway"), }, } xRes, err := xTranslator.Translate(val) diff --git a/internal/infrastructure/kubernetes/applier/apply.go b/internal/infrastructure/kubernetes/applier/apply.go new file mode 100644 index 00000000000..6aa7f751d6c --- /dev/null +++ b/internal/infrastructure/kubernetes/applier/apply.go @@ -0,0 +1,187 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package applier + +import ( + "context" + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Instance struct { + Client client.Client +} + +func New(cli client.Client) *Instance { + return &Instance{ + Client: cli, + } +} + +func (i *Instance) CreateOrUpdateConfigMap(ctx context.Context, cm *corev1.ConfigMap) error { + current := &corev1.ConfigMap{} + key := types.NamespacedName{ + Namespace: cm.Namespace, + Name: cm.Name, + } + + if err := i.Client.Get(ctx, key, current); err != nil { + // Create if not found. + if kerrors.IsNotFound(err) { + if err := i.Client.Create(ctx, cm); err != nil { + return fmt.Errorf("failed to create configmap %s/%s: %w", cm.Namespace, cm.Name, err) + } + } + } else { + // Update if current value is different. + if !reflect.DeepEqual(cm.Data, current.Data) { + cm.ResourceVersion = current.ResourceVersion + cm.UID = current.UID + if err := i.Client.Update(ctx, cm); err != nil { + return fmt.Errorf("failed to update configmap %s/%s: %w", cm.Namespace, cm.Name, err) + } + } + } + + return nil +} + +func (i *Instance) DeleteConfigMap(ctx context.Context, cm *corev1.ConfigMap) error { + if err := i.Client.Delete(ctx, cm); err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete configmap %s/%s: %w", cm.Namespace, cm.Name, err) + } + + return nil +} + +func (i *Instance) CreateOrUpdateDeployment(ctx context.Context, deployment *appsv1.Deployment) error { + current := &appsv1.Deployment{} + key := types.NamespacedName{ + Namespace: deployment.Namespace, + Name: deployment.Name, + } + if err := i.Client.Get(ctx, key, current); err != nil { + // Create if not found. + if kerrors.IsNotFound(err) { + if err := i.Client.Create(ctx, deployment); err != nil { + return fmt.Errorf("failed to create deployment %s/%s: %w", + deployment.Namespace, deployment.Name, err) + } + } + } else { + // Update if current value is different. + if !reflect.DeepEqual(deployment.Spec, current.Spec) { + deployment.ResourceVersion = current.ResourceVersion + deployment.UID = current.UID + if err := i.Client.Update(ctx, deployment); err != nil { + return fmt.Errorf("failed to update deployment %s/%s: %w", + deployment.Namespace, deployment.Name, err) + } + } + } + + return nil +} + +func (i *Instance) DeleteDeployment(ctx context.Context, deploy *appsv1.Deployment) error { + if err := i.Client.Delete(ctx, deploy); err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + } + return nil +} + +func (i *Instance) CreateOrUpdateService(ctx context.Context, svc *corev1.Service) error { + current := &corev1.Service{} + key := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + + if err := i.Client.Get(ctx, key, current); err != nil { + // Create if not found. + if kerrors.IsNotFound(err) { + if err := i.Client.Create(ctx, svc); err != nil { + return fmt.Errorf("failed to create service %s/%s: %w", + svc.Namespace, svc.Name, err) + } + } + } else { + // Update if current value is different. + if !reflect.DeepEqual(svc.Spec, current.Spec) { + svc.ResourceVersion = current.ResourceVersion + svc.UID = current.UID + if err := i.Client.Update(ctx, svc); err != nil { + return fmt.Errorf("failed to update service %s/%s: %w", + svc.Namespace, svc.Name, err) + } + } + } + + return nil +} + +func (i *Instance) DeleteService(ctx context.Context, svc *corev1.Service) error { + if err := i.Client.Delete(ctx, svc); err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete service %s/%s: %w", svc.Namespace, svc.Name, err) + } + + return nil +} + +func (i *Instance) CreateOrUpdateServiceAccount(ctx context.Context, sa *corev1.ServiceAccount) error { + current := &corev1.ServiceAccount{} + key := types.NamespacedName{ + Namespace: sa.Namespace, + Name: sa.Name, + } + + if err := i.Client.Get(ctx, key, current); err != nil { + if kerrors.IsNotFound(err) { + // Create if it does not exist. + if err := i.Client.Create(ctx, sa); err != nil { + return fmt.Errorf("failed to create serviceaccount %s/%s: %w", + sa.Namespace, sa.Name, err) + } + } + } else { + // Since the ServiceAccount does not have a specific Spec field to compare + // just perform an update for now. + sa.ResourceVersion = current.ResourceVersion + sa.UID = current.UID + if err := i.Client.Update(ctx, sa); err != nil { + return fmt.Errorf("failed to update serviceaccount %s/%s: %w", + sa.Namespace, sa.Name, err) + } + } + + return nil +} + +func (i *Instance) DeleteServiceAccount(ctx context.Context, sa *corev1.ServiceAccount) error { + if err := i.Client.Delete(ctx, sa); err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete serviceaccount %s/%s: %w", sa.Namespace, sa.Name, err) + } + + return nil +} diff --git a/internal/infrastructure/kubernetes/configmap.go b/internal/infrastructure/kubernetes/configmap.go deleted file mode 100644 index 0a101b8cfee..00000000000 --- a/internal/infrastructure/kubernetes/configmap.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - "reflect" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" -) - -const ( - sdsCAFilename = "xds-trusted-ca.json" - sdsCertFilename = "xds-certificate.json" - // xdsTLSCertFilename is the fully qualified path of the file containing Envoy's - // xDS server TLS certificate. - xdsTLSCertFilename = "/certs/tls.crt" - // xdsTLSKeyFilename is the fully qualified path of the file containing Envoy's - // xDS server TLS key. - xdsTLSKeyFilename = "/certs/tls.key" - // xdsTLSCaFilename is the fully qualified path of the file containing Envoy's - // trusted CA certificate. - xdsTLSCaFilename = "/certs/ca.crt" -) - -var ( - // xDS certificate rotation is supported by using SDS path-based resource files. - sdsCAConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ - `"name":"xds_trusted_ca","validation_context":{"trusted_ca":{"filename":"%s"},`+ - `"match_typed_subject_alt_names":[{"san_type":"DNS","matcher":{"exact":"envoy-gateway"}}]}}]}`, xdsTLSCaFilename) - sdsCertConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ - `"name":"xds_certificate","tls_certificate":{"certificate_chain":{"filename":"%s"},`+ - `"private_key":{"filename":"%s"}}}]}`, xdsTLSCertFilename, xdsTLSKeyFilename) -) - -func (i *Infra) createOrUpdateConfigMap(ctx context.Context, cm *corev1.ConfigMap) error { - current := &corev1.ConfigMap{} - key := types.NamespacedName{ - Namespace: cm.Namespace, - Name: cm.Name, - } - - if err := i.Client.Get(ctx, key, current); err != nil { - // Create if not found. - if kerrors.IsNotFound(err) { - if err := i.Client.Create(ctx, cm); err != nil { - return fmt.Errorf("failed to create configmap %s/%s: %w", cm.Namespace, cm.Name, err) - } - } - } else { - // Update if current value is different. - if !reflect.DeepEqual(cm.Data, current.Data) { - cm.ResourceVersion = current.ResourceVersion - cm.UID = current.UID - if err := i.Client.Update(ctx, cm); err != nil { - return fmt.Errorf("failed to update configmap %s/%s: %w", cm.Namespace, cm.Name, err) - } - } - } - - return nil -} - -func (i *Infra) deleteConfigMap(ctx context.Context, cm *corev1.ConfigMap) error { - if err := i.Client.Delete(ctx, cm); err != nil { - if kerrors.IsNotFound(err) { - return nil - } - return fmt.Errorf("failed to delete configmap %s/%s: %w", cm.Namespace, cm.Name, err) - } - - return nil -} diff --git a/internal/infrastructure/kubernetes/deployment.go b/internal/infrastructure/kubernetes/deployment.go deleted file mode 100644 index d8a4eeb59aa..00000000000 --- a/internal/infrastructure/kubernetes/deployment.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - "reflect" - - appsv1 "k8s.io/api/apps/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" -) - -func (i *Infra) createOrUpdateDeployment(ctx context.Context, deployment *appsv1.Deployment) error { - current := &appsv1.Deployment{} - key := types.NamespacedName{ - Namespace: deployment.Namespace, - Name: deployment.Name, - } - if err := i.Client.Get(ctx, key, current); err != nil { - // Create if not found. - if kerrors.IsNotFound(err) { - if err := i.Client.Create(ctx, deployment); err != nil { - return fmt.Errorf("failed to create deployment %s/%s: %w", - deployment.Namespace, deployment.Name, err) - } - } - } else { - // Update if current value is different. - if !reflect.DeepEqual(deployment.Spec, current.Spec) { - deployment.ResourceVersion = current.ResourceVersion - deployment.UID = current.UID - if err := i.Client.Update(ctx, deployment); err != nil { - return fmt.Errorf("failed to update deployment %s/%s: %w", - deployment.Namespace, deployment.Name, err) - } - } - } - - return nil -} - -func (i *Infra) deleteDeployment(ctx context.Context, deploy *appsv1.Deployment) error { - if err := i.Client.Delete(ctx, deploy); err != nil { - if kerrors.IsNotFound(err) { - return nil - } - return fmt.Errorf("failed to delete deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - } - return nil -} diff --git a/internal/infrastructure/kubernetes/deployment_test.go b/internal/infrastructure/kubernetes/deployment_test.go deleted file mode 100644 index 106f40993bc..00000000000 --- a/internal/infrastructure/kubernetes/deployment_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "testing" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" -) - -func checkEnvVar(t *testing.T, deploy *appsv1.Deployment, container, name string) { - t.Helper() - - for i, c := range deploy.Spec.Template.Spec.Containers { - if c.Name == container { - for _, envVar := range deploy.Spec.Template.Spec.Containers[i].Env { - if envVar.Name == name { - return - } - } - } - } - - t.Errorf("deployment is missing environment variable %q", name) -} - -func checkContainer(t *testing.T, deploy *appsv1.Deployment, name string, expect bool) *corev1.Container { - t.Helper() - - if deploy.Spec.Template.Spec.Containers == nil { - t.Error("deployment has no containers") - } - - for _, container := range deploy.Spec.Template.Spec.Containers { - if container.Name == name { - if expect { - return &container - } - t.Errorf("deployment has unexpected %q container", name) - } - } - - if expect { - t.Errorf("deployment has no %q container", name) - } - return nil -} - -func checkContainerHasArg(t *testing.T, container *corev1.Container, arg string) { - t.Helper() - - for _, a := range container.Args { - if a == arg { - return - } - } - t.Errorf("container is missing argument %q", arg) -} - -func checkLabels(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) { - t.Helper() - - if apiequality.Semantic.DeepEqual(deploy.Labels, expected) { - return - } - - t.Errorf("deployment has unexpected %q labels", deploy.Labels) -} - -func checkContainerHasPort(t *testing.T, deploy *appsv1.Deployment, port int32) { - t.Helper() - - for _, c := range deploy.Spec.Template.Spec.Containers { - for _, p := range c.Ports { - if p.ContainerPort == port { - return - } - } - } - t.Errorf("container is missing containerPort %q", port) -} - -func checkPodAnnotations(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) { - t.Helper() - - if apiequality.Semantic.DeepEqual(deploy.Spec.Template.Annotations, expected) { - return - } - - t.Errorf("deployment has unexpected %q pod annotations ", deploy.Spec.Template.Annotations) -} - -func checkPodSecurityContext(t *testing.T, deploy *appsv1.Deployment, expected *corev1.PodSecurityContext) { - t.Helper() - - if apiequality.Semantic.DeepEqual(deploy.Spec.Template.Spec.SecurityContext, expected) { - return - } - - t.Errorf("deployment has unexpected %q pod annotations ", deploy.Spec.Template.Spec.SecurityContext) -} - -func checkContainerSecurityContext(t *testing.T, container *corev1.Container, expected *corev1.SecurityContext) { - t.Helper() - - if apiequality.Semantic.DeepEqual(container.SecurityContext, expected) { - return - } - - t.Errorf("container has unexpected %q pod annotations ", container.SecurityContext) -} - -func checkContainerImage(t *testing.T, container *corev1.Container, image string) { - t.Helper() - - if container.Image == image { - return - } - t.Errorf("container is missing image %q", image) -} - -func checkContainerResources(t *testing.T, container *corev1.Container, expected *corev1.ResourceRequirements) { - t.Helper() - if apiequality.Semantic.DeepEqual(&container.Resources, expected) { - return - } - - t.Errorf("container has unexpected %q resources ", expected) -} diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index 7ae9130842e..ee81eba89d5 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -6,15 +6,35 @@ package kubernetes import ( - "fmt" + "context" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/provider/utils" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/applier" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" ) +var ( + _ ResourceRender = &proxy.ResourceRender{} + _ ResourceRender = &ratelimit.ResourceRender{} +) + +// ResourceRender renders Kubernetes infrastructure resources +// based on Infra IR resources. +type ResourceRender interface { + Name() string + ServiceAccount() (*corev1.ServiceAccount, error) + Service() (*corev1.Service, error) + ConfigMap() (*corev1.ConfigMap, error) + Deployment() (*appsv1.Deployment, error) +} + // Infra manages the creation and deletion of Kubernetes infrastructure // based on Infra IR resources. type Infra struct { @@ -25,6 +45,7 @@ type Infra struct { // EnvoyGateway is the configuration used to startup Envoy Gateway. EnvoyGateway *v1alpha1.EnvoyGateway + applier *applier.Instance } // NewInfra returns a new Infra. @@ -33,11 +54,137 @@ func NewInfra(cli client.Client, cfg *config.Server) *Infra { Client: cli, Namespace: cfg.Namespace, EnvoyGateway: cfg.EnvoyGateway, + applier: applier.New(cli), } } -// expectedResourceHashedName returns hashed resource name. -func expectedResourceHashedName(name string) string { - hashedName := utils.GetHashedName(name) - return fmt.Sprintf("%s-%s", config.EnvoyPrefix, hashedName) +func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { + if err := i.createOrUpdateServiceAccount(ctx, r); err != nil { + return err + } + + if err := i.createOrUpdateConfigMap(ctx, r); err != nil { + return err + } + + if err := i.createOrUpdateDeployment(ctx, r); err != nil { + return err + } + + if err := i.createOrUpdateService(ctx, r); err != nil { + return err + } + + return nil +} + +// createOrUpdateServiceAccount creates a ServiceAccount in the kube api server based on the +// provided ResourceRender, if it doesn't exist and updates it if it does. +func (i *Infra) createOrUpdateServiceAccount(ctx context.Context, r ResourceRender) error { + sa, err := r.ServiceAccount() + if err != nil { + return err + } + return i.applier.CreateOrUpdateServiceAccount(ctx, sa) +} + +// createOrUpdateConfigMap creates a ConfigMap in the Kube api server based on the provided +// ResourceRender, if it doesn't exist and updates it if it does. +func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender) error { + cm, err := r.ConfigMap() + if err != nil { + return err + } + + return i.applier.CreateOrUpdateConfigMap(ctx, cm) +} + +// createOrUpdateDeployment creates a Deployment in the kube api server based on the provided +// ResourceRender, if it doesn't exist and updates it if it does. +func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) error { + deployment, err := r.Deployment() + if err != nil { + return err + } + return i.applier.CreateOrUpdateDeployment(ctx, deployment) +} + +// createOrUpdateRateLimitService creates a Service in the kube api server based on the provided ResourceRender, +// if it doesn't exist or updates it if it does. +func (i *Infra) createOrUpdateService(ctx context.Context, r ResourceRender) error { + svc, err := r.Service() + if err != nil { + return err + } + + return i.applier.CreateOrUpdateService(ctx, svc) +} + +func (i *Infra) delete(ctx context.Context, r ResourceRender) error { + if err := i.deleteServiceAccount(ctx, r); err != nil { + return err + } + + if err := i.deleteConfigMap(ctx, r); err != nil { + return err + } + + if err := i.deleteDeployment(ctx, r); err != nil { + return err + } + + if err := i.deleteService(ctx, r); err != nil { + return err + } + + return nil +} + +// deleteServiceAccount deletes the ServiceAccount in the kube api server, +// if it exists. +func (i *Infra) deleteServiceAccount(ctx context.Context, r ResourceRender) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.applier.DeleteServiceAccount(ctx, sa) +} + +// deleteDeployment deletes the Envoy Deployment in the kube api server, if it exists. +func (i *Infra) deleteDeployment(ctx context.Context, r ResourceRender) error { + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.applier.DeleteDeployment(ctx, deploy) +} + +// deleteConfigMap deletes the ConfigMap in the kube api server, if it exists. +func (i *Infra) deleteConfigMap(ctx context.Context, r ResourceRender) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.applier.DeleteConfigMap(ctx, cm) +} + +// deleteService deletes the Service in the kube api server, if it exists. +func (i *Infra) deleteService(ctx context.Context, r ResourceRender) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.applier.DeleteService(ctx, svc) } diff --git a/internal/infrastructure/kubernetes/labels.go b/internal/infrastructure/kubernetes/labels.go deleted file mode 100644 index 99596eb5ae0..00000000000 --- a/internal/infrastructure/kubernetes/labels.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// getSelector returns a label selector used to select resources -// based on the provided labels. -func getSelector(labels map[string]string) *metav1.LabelSelector { - return &metav1.LabelSelector{ - MatchLabels: labels, - } -} diff --git a/internal/infrastructure/kubernetes/proxy_deployment.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go similarity index 56% rename from internal/infrastructure/kubernetes/proxy_deployment.go rename to internal/infrastructure/kubernetes/proxy/resource_provider.go index 97aa98f7d4f..84118e85606 100644 --- a/internal/infrastructure/kubernetes/proxy_deployment.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -3,19 +3,20 @@ // The full text of the Apache license is available in the LICENSE file at // the root of the repo. -package kubernetes +package proxy import ( - "context" "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/xds/bootstrap" ) @@ -27,34 +28,148 @@ const ( envoyNsEnvVar = "ENVOY_GATEWAY_NAMESPACE" // envoyPodEnvVar is the name of the Envoy pod name environment variable. envoyPodEnvVar = "ENVOY_POD_NAME" - // envoyHTTPPort is the container port number of Envoy's HTTP endpoint. - envoyHTTPPort = int32(8080) - // envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint. - envoyHTTPSPort = int32(8443) ) -// expectedProxyDeployment returns the expected Deployment based on the provided infra. -func (i *Infra) expectedProxyDeployment(infra *ir.Infra) (*appsv1.Deployment, error) { +type ResourceRender struct { + infra *ir.Infra + + // Namespace is the Namespace used for managed infra. + Namespace string +} + +func NewResourceRender(ns string, infra *ir.Infra) *ResourceRender { + return &ResourceRender{ + Namespace: ns, + infra: infra, + } +} + +func (r *ResourceRender) Name() string { + return utils.ExpectedResourceHashedName(r.infra.Proxy.Name) +} + +// ServiceAccount returns the expected proxy serviceAccount. +func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { + // Set the labels based on the owning gateway name. + labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { + return nil, fmt.Errorf("missing owning gateway labels") + } + + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Labels: labels, + }, + }, nil +} + +// Service returns the expected Service based on the provided infra. +func (r *ResourceRender) Service() (*corev1.Service, error) { + var ports []corev1.ServicePort + for _, listener := range r.infra.Proxy.Listeners { + for _, port := range listener.Ports { + target := intstr.IntOrString{IntVal: port.ContainerPort} + protocol := corev1.ProtocolTCP + if port.Protocol == ir.UDPProtocolType { + protocol = corev1.ProtocolUDP + } + p := corev1.ServicePort{ + Name: port.Name, + Protocol: protocol, + Port: port.ServicePort, + TargetPort: target, + } + ports = append(ports, p) + } + } + + // Set the labels based on the owning gatewayclass name. + labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { + return nil, fmt.Errorf("missing owning gateway labels") + } + + // Get annotations + var annotations map[string]string + provider := r.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() + envoyServiceConfig := provider.GetEnvoyProxyKubeProvider().EnvoyService + if envoyServiceConfig.Annotations != nil { + annotations = envoyServiceConfig.Annotations + } + serviceSpec := utils.ExpectedServiceSpec(envoyServiceConfig.Type) + serviceSpec.Ports = ports + serviceSpec.Selector = utils.GetSelector(labels).MatchLabels + + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Labels: labels, + Annotations: annotations, + }, + Spec: serviceSpec, + } + + return svc, nil +} + +// ConfigMap returns the expected ConfigMap based on the provided infra. +func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { + // Set the labels based on the owning gateway name. + labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { + return nil, fmt.Errorf("missing owning gateway labels") + } + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Labels: labels, + }, + Data: map[string]string{ + utils.SdsCAFilename: utils.SdsCAConfigMapData, + utils.SdsCertFilename: utils.SdsCertConfigMapData, + }, + }, nil +} + +// ExpectedDeployment returns the expected Deployment based on the provided infra. +func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { // Get the EnvoyProxy config to configure the deployment. - provider := infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() + provider := r.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() if provider.Type != egcfgv1a1.ProviderTypeKubernetes { return nil, fmt.Errorf("invalid provider type %v for Kubernetes infra manager", provider.Type) } deploymentConfig := provider.GetEnvoyProxyKubeProvider().EnvoyDeployment // Get expected bootstrap configurations rendered ProxyContainers - containers, err := expectedProxyContainers(infra, deploymentConfig) + containers, err := expectedProxyContainers(r.infra, deploymentConfig) if err != nil { return nil, err } // Set the labels based on the owning gateway name. - labels := envoyLabels(infra.GetProxyInfra().GetProxyMetadata().Labels) + labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { return nil, fmt.Errorf("missing owning gateway labels") } - selector := getSelector(labels) + selector := utils.GetSelector(labels) // Get annotations var annotations map[string]string @@ -68,8 +183,8 @@ func (i *Infra) expectedProxyDeployment(infra *ir.Infra) (*appsv1.Deployment, er APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), + Namespace: r.Namespace, + Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), Labels: labels, }, Spec: appsv1.DeploymentSpec{ @@ -82,7 +197,7 @@ func (i *Infra) expectedProxyDeployment(infra *ir.Infra) (*appsv1.Deployment, er }, Spec: corev1.PodSpec{ Containers: containers, - ServiceAccountName: expectedResourceHashedName(infra.Proxy.Name), + ServiceAccountName: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), AutomountServiceAccountToken: pointer.Bool(false), TerminationGracePeriodSeconds: pointer.Int64(int64(300)), DNSPolicy: corev1.DNSClusterFirst, @@ -103,16 +218,16 @@ func (i *Infra) expectedProxyDeployment(infra *ir.Infra) (*appsv1.Deployment, er VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: expectedResourceHashedName(infra.Proxy.Name), + Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), }, Items: []corev1.KeyToPath{ { - Key: sdsCAFilename, - Path: sdsCAFilename, + Key: utils.SdsCAFilename, + Path: utils.SdsCAFilename, }, { - Key: sdsCertFilename, - Path: sdsCertFilename, + Key: utils.SdsCertFilename, + Path: utils.SdsCertFilename, }, }, DefaultMode: pointer.Int32(int32(420)), @@ -224,25 +339,3 @@ func expectedProxyContainers(infra *ir.Infra, deploymentConfig *egcfgv1a1.Kubern return containers, nil } - -// createOrUpdateProxyDeployment creates a Deployment in the kube api server based on the provided -// infra, if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateProxyDeployment(ctx context.Context, infra *ir.Infra) error { - deploy, err := i.expectedProxyDeployment(infra) - if err != nil { - return err - } - return i.createOrUpdateDeployment(ctx, deploy) -} - -// deleteProxyDeployment deletes the Envoy Deployment in the kube api server, if it exists. -func (i *Infra) deleteProxyDeployment(ctx context.Context, infra *ir.Infra) error { - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - }, - } - - return i.deleteDeployment(ctx, deploy) -} diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go new file mode 100644 index 00000000000..2aea42e9ae1 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -0,0 +1,251 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package proxy + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/pointer" + "sigs.k8s.io/yaml" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/ir" +) + +const ( + // envoyHTTPPort is the container port number of Envoy's HTTP endpoint. + envoyHTTPPort = int32(8080) + // envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint. + envoyHTTPSPort = int32(8443) +) + +func newTestInfra() *ir.Infra { + i := ir.NewInfra() + + i.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" + i.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = i.Proxy.Name + i.Proxy.Listeners = []ir.ProxyListener{ + { + Ports: []ir.ListenerPort{ + { + Name: "EnvoyHTTPPort", + Protocol: ir.TCPProtocolType, + ContainerPort: envoyHTTPPort, + }, + { + Name: "EnvoyHTTPSPort", + Protocol: ir.TCPProtocolType, + ContainerPort: envoyHTTPSPort, + }, + }, + }, + } + + return i +} + +func TestDeployment(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + infra *ir.Infra + deploy *egcfgv1a1.KubernetesDeploymentSpec + bootstrap *string + }{ + { + caseName: "default", + infra: newTestInfra(), + deploy: nil, + }, + { + caseName: "custom", + infra: newTestInfra(), + deploy: &egcfgv1a1.KubernetesDeploymentSpec{ + Replicas: pointer.Int32(2), + Pod: &egcfgv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64(1000), + }, + }, + Container: &egcfgv1a1.KubernetesContainerSpec{ + Image: pointer.String("envoyproxy/envoy:v1.2.3"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.Bool(true), + }, + }, + }, + }, + { + caseName: "bootstrap", + infra: newTestInfra(), + deploy: nil, + bootstrap: pointer.String(`test bootstrap config`), + }, + // TODO: add test cases for custom ProxyLogging + } + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + kube := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider().GetEnvoyProxyKubeProvider() + if tc.deploy != nil { + kube.EnvoyDeployment = tc.deploy + } + + if tc.bootstrap != nil && *tc.bootstrap != "" { + tc.infra.Proxy.Config.Spec.Bootstrap = tc.bootstrap + } + + r := NewResourceRender(cfg.Namespace, tc.infra) + dp, err := r.Deployment() + require.NoError(t, err) + + expected, err := loadDeployment(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, expected, dp) + }) + } +} + +func loadDeployment(caseName string) (*appsv1.Deployment, error) { + deploymentYAML, err := os.ReadFile(fmt.Sprintf("testdata/deployments/%s.yaml", caseName)) + if err != nil { + return nil, err + } + deployment := &appsv1.Deployment{} + _ = yaml.Unmarshal(deploymentYAML, deployment) + return deployment, nil +} + +func TestService(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + svcType := egcfgv1a1.ServiceTypeClusterIP + cases := []struct { + caseName string + infra *ir.Infra + service *egcfgv1a1.KubernetesServiceSpec + }{ + { + caseName: "default", + infra: newTestInfra(), + service: nil, + }, + { + caseName: "custom", + infra: newTestInfra(), + service: &egcfgv1a1.KubernetesServiceSpec{ + Annotations: map[string]string{ + "key1": "value1", + }, + Type: &svcType, + }, + }, + } + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + provider := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider().GetEnvoyProxyKubeProvider() + if tc.service != nil { + provider.EnvoyService = tc.service + } + + r := NewResourceRender(cfg.Namespace, tc.infra) + svc, err := r.Service() + require.NoError(t, err) + + expected, err := loadService(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, expected, svc) + }) + } +} + +func loadService(caseName string) (*corev1.Service, error) { + serviceYAML, err := os.ReadFile(fmt.Sprintf("testdata/services/%s.yaml", caseName)) + if err != nil { + return nil, err + } + svc := &corev1.Service{} + _ = yaml.Unmarshal(serviceYAML, svc) + return svc, nil +} + +func TestConfigMap(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + infra := newTestInfra() + + r := NewResourceRender(cfg.Namespace, infra) + cm, err := r.ConfigMap() + require.NoError(t, err) + + expected, err := loadConfigmap() + require.NoError(t, err) + + assert.Equal(t, expected, cm) +} + +func loadConfigmap() (*corev1.ConfigMap, error) { + cmYAML, err := os.ReadFile("testdata/configmap.yaml") + if err != nil { + return nil, err + } + cm := &corev1.ConfigMap{} + _ = yaml.Unmarshal(cmYAML, cm) + return cm, nil +} + +func TestServiceAccount(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + infra := newTestInfra() + + r := NewResourceRender(cfg.Namespace, infra) + sa, err := r.ServiceAccount() + require.NoError(t, err) + + expected, err := loadServiceAccount() + require.NoError(t, err) + + assert.Equal(t, expected, sa) +} + +func loadServiceAccount() (*corev1.ServiceAccount, error) { + saYAML, err := os.ReadFile("testdata/serviceaccount.yaml") + if err != nil { + return nil, err + } + sa := &corev1.ServiceAccount{} + _ = yaml.Unmarshal(saYAML, sa) + return sa, nil +} diff --git a/internal/infrastructure/kubernetes/proxy/testdata/configmap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/configmap.yaml new file mode 100644 index 00000000000..61e4d0f571c --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +data: + xds-certificate.json: '{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret","name":"xds_certificate","tls_certificate":{"certificate_chain":{"filename":"/certs/tls.crt"},"private_key":{"filename":"/certs/tls.key"}}}]}' + xds-trusted-ca.json: '{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret","name":"xds_trusted_ca","validation_context":{"trusted_ca":{"filename":"/certs/ca.crt"},"match_typed_subject_alt_names":[{"san_type":"DNS","matcher":{"exact":"envoy-gateway"}}]}}]}' diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml new file mode 100644 index 00000000000..291ea95ef35 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +spec: + replicas: 1 + selector: + matchLabels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - --config-yaml test bootstrap config + - --log-level info + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy-dev:latest + imagePullPolicy: IfNotPresent + name: envoy + ports: + - containerPort: 8080 + name: EnvoyHTTPPort + protocol: TCP + - containerPort: 8443 + name: EnvoyHTTPSPort + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-64656661 + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-64656661 + optional: false + name: sds diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml new file mode 100644 index 00000000000..198625f4f8a --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -0,0 +1,160 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +spec: + replicas: 2 + selector: + matchLabels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + annotations: + prometheus.io/scrape: "true" + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + cds_config: + ads: {} + static_resources: + clusters: + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + "explicit_http_config": + "http2_protocol_options": {} + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + layered_runtime: + layers: + - name: runtime-0 + rtds_layer: + rtds_config: + ads: {} + name: runtime-0 + - --log-level info + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:v1.2.3 + imagePullPolicy: IfNotPresent + name: envoy + ports: + - containerPort: 8080 + name: EnvoyHTTPPort + protocol: TCP + - containerPort: 8443 + name: EnvoyHTTPSPort + protocol: TCP + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + securityContext: + privileged: true + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-64656661 + terminationGracePeriodSeconds: 300 + securityContext: + runAsUser: 1000 + volumes: + - name: certs + secret: + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-64656661 + optional: false + name: sds diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml new file mode 100644 index 00000000000..e6a296fe720 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -0,0 +1,151 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +spec: + replicas: 1 + selector: + matchLabels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + cds_config: + ads: {} + static_resources: + clusters: + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + "explicit_http_config": + "http2_protocol_options": {} + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + layered_runtime: + layers: + - name: runtime-0 + rtds_layer: + rtds_config: + ads: {} + name: runtime-0 + - --log-level info + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy-dev:latest + imagePullPolicy: IfNotPresent + name: envoy + ports: + - containerPort: 8080 + name: EnvoyHTTPPort + protocol: TCP + - containerPort: 8443 + name: EnvoyHTTPSPort + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-64656661 + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-64656661 + optional: false + name: sds diff --git a/internal/infrastructure/kubernetes/proxy/testdata/serviceaccount.yaml b/internal/infrastructure/kubernetes/proxy/testdata/serviceaccount.yaml new file mode 100644 index 00000000000..f6fdb690d21 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system diff --git a/internal/infrastructure/kubernetes/proxy/testdata/services/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/services/custom.yaml new file mode 100644 index 00000000000..140dc728103 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/services/custom.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + key1: value1 + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +spec: + ports: + - name: EnvoyHTTPPort + port: 0 + protocol: TCP + targetPort: 8080 + - name: EnvoyHTTPSPort + port: 0 + protocol: TCP + targetPort: 8443 + selector: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + sessionAffinity: None + type: ClusterIP diff --git a/internal/infrastructure/kubernetes/proxy/testdata/services/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/services/default.yaml new file mode 100644 index 00000000000..d46ed693a90 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/services/default.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-64656661 + namespace: envoy-gateway-system +spec: + externalTrafficPolicy: Local + ports: + - name: EnvoyHTTPPort + port: 0 + protocol: TCP + targetPort: 8080 + - name: EnvoyHTTPSPort + port: 0 + protocol: TCP + targetPort: 8443 + selector: + app.gateway.envoyproxy.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + sessionAffinity: None + type: LoadBalancer diff --git a/internal/infrastructure/kubernetes/proxy_configmap.go b/internal/infrastructure/kubernetes/proxy_configmap.go deleted file mode 100644 index f61835ed1e8..00000000000 --- a/internal/infrastructure/kubernetes/proxy_configmap.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedProxyConfigMap returns the expected ConfigMap based on the provided infra. -func (i *Infra) expectedProxyConfigMap(infra *ir.Infra) (*corev1.ConfigMap, error) { - // Set the labels based on the owning gateway name. - labels := envoyLabels(infra.GetProxyInfra().GetProxyMetadata().Labels) - if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { - return nil, fmt.Errorf("missing owning gateway labels") - } - - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - Labels: labels, - }, - Data: map[string]string{ - sdsCAFilename: sdsCAConfigMapData, - sdsCertFilename: sdsCertConfigMapData, - }, - }, nil -} - -// createOrUpdateProxyConfigMap creates a ConfigMap in the Kube api server based on the provided -// infra, if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateProxyConfigMap(ctx context.Context, infra *ir.Infra) error { - cm, err := i.expectedProxyConfigMap(infra) - if err != nil { - return err - } - - return i.createOrUpdateConfigMap(ctx, cm) -} - -// deleteProxyConfigMap deletes the Envoy ConfigMap in the kube api server, if it exists. -func (i *Infra) deleteProxyConfigMap(ctx context.Context, infra *ir.Infra) error { - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - }, - } - - return i.deleteConfigMap(ctx, cm) -} diff --git a/internal/infrastructure/kubernetes/proxy_configmap_test.go b/internal/infrastructure/kubernetes/proxy_configmap_test.go index 3b49e30b279..1a7e775978d 100644 --- a/internal/infrastructure/kubernetes/proxy_configmap_test.go +++ b/internal/infrastructure/kubernetes/proxy_configmap_test.go @@ -20,48 +20,15 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) -func TestExpectedProxyConfigMap(t *testing.T) { - // Setup the infra. - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - cfg, err := config.New() - require.NoError(t, err) - - kube := NewInfra(cli, cfg) - infra := ir.NewInfra() - - infra.Proxy.Name = "test" - - // An infra without Gateway owner labels should trigger - // an error. - _, err = kube.expectedProxyConfigMap(infra) - require.NotNil(t, err) - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - cm, err := kube.expectedProxyConfigMap(infra) - require.NoError(t, err) - - require.Equal(t, "envoy-test-74657374", cm.Name) - require.Equal(t, "envoy-gateway-system", cm.Namespace) - require.Contains(t, cm.Data, sdsCAFilename) - assert.Equal(t, sdsCAConfigMapData, cm.Data[sdsCAFilename]) - require.Contains(t, cm.Data, sdsCertFilename) - assert.Equal(t, sdsCertConfigMapData, cm.Data[sdsCertFilename]) - - wantLabels := envoyAppLabel() - wantLabels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - wantLabels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - assert.True(t, apiequality.Semantic.DeepEqual(wantLabels, cm.Labels)) -} - func TestCreateOrUpdateProxyConfigMap(t *testing.T) { cfg, err := config.New() require.NoError(t, err) - kube := NewInfra(nil, cfg) + infra := ir.NewInfra() infra.Proxy.Name = "test" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" @@ -84,7 +51,10 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) { gatewayapi.OwningGatewayNameLabel: "test", }, }, - Data: map[string]string{sdsCAFilename: sdsCAConfigMapData, sdsCertFilename: sdsCertConfigMapData}, + Data: map[string]string{ + utils.SdsCAFilename: utils.SdsCAConfigMapData, + utils.SdsCertFilename: utils.SdsCertConfigMapData, + }, }, }, { @@ -111,7 +81,10 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) { gatewayapi.OwningGatewayNameLabel: "test", }, }, - Data: map[string]string{sdsCAFilename: sdsCAConfigMapData, sdsCertFilename: sdsCertConfigMapData}, + Data: map[string]string{ + utils.SdsCAFilename: utils.SdsCAConfigMapData, + utils.SdsCertFilename: utils.SdsCertConfigMapData, + }, }, }, } @@ -119,12 +92,15 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateProxyConfigMap(context.Background(), infra) + kube := NewInfra(cli, cfg) + r := proxy.NewResourceRender(kube.Namespace, infra) + err := kube.createOrUpdateConfigMap(context.Background(), r) require.NoError(t, err) actual := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -182,7 +158,8 @@ func TestDeleteConfigProxyMap(t *testing.T) { infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - err = kube.deleteProxyConfigMap(context.Background(), infra) + r := proxy.NewResourceRender(kube.Namespace, infra) + err = kube.deleteConfigMap(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/proxy_deployment_test.go b/internal/infrastructure/kubernetes/proxy_deployment_test.go index 4a50ac543cc..29d8d20d8a7 100644 --- a/internal/infrastructure/kubernetes/proxy_deployment_test.go +++ b/internal/infrastructure/kubernetes/proxy_deployment_test.go @@ -7,14 +7,10 @@ package kubernetes import ( "context" - "fmt" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,234 +20,15 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" - "github.com/envoyproxy/gateway/internal/xds/bootstrap" ) -func testExpectedProxyDeployment(t *testing.T, - infra *ir.Infra, - expectedResources *corev1.ResourceRequirements, - expectedPodSecurityContext *corev1.PodSecurityContext, - expectedSecurityContext *corev1.SecurityContext) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - infra.Proxy.Listeners = []ir.ProxyListener{ - { - Ports: []ir.ListenerPort{ - { - Name: "EnvoyHTTPPort", - Protocol: ir.TCPProtocolType, - ContainerPort: envoyHTTPPort, - }, - { - Name: "EnvoyHTTPSPort", - Protocol: ir.TCPProtocolType, - ContainerPort: envoyHTTPSPort, - }, - }, - }, - } - - deploy, err := kube.expectedProxyDeployment(infra) - require.NoError(t, err) - - // Check the deployment name is as expected. - assert.Equal(t, deploy.Name, expectedResourceHashedName(infra.Proxy.Name)) - - // Check container details, i.e. env vars, labels, etc. for the deployment are as expected. - container := checkContainer(t, deploy, envoyContainerName, true) - checkContainerResources(t, container, expectedResources) - checkContainerSecurityContext(t, container, expectedSecurityContext) - checkPodSecurityContext(t, deploy, expectedPodSecurityContext) - checkEnvVar(t, deploy, envoyContainerName, envoyNsEnvVar) - checkEnvVar(t, deploy, envoyContainerName, envoyPodEnvVar) - checkLabels(t, deploy, deploy.Labels) - - // Create a bootstrap config, render it into an arg, and ensure it's as expected. - bstrap, err := bootstrap.GetRenderedBootstrapConfig() - require.NoError(t, err) - checkContainerHasArg(t, container, fmt.Sprintf("--config-yaml %s", bstrap)) - - // Check container ports for the deployment are as expected. - ports := []int32{envoyHTTPPort, envoyHTTPSPort} - for _, port := range ports { - checkContainerHasPort(t, deploy, port) - } - - // Set the deployment replicas. - repl := int32(2) - infra.Proxy.GetProxyConfig().GetEnvoyProxyProvider().GetEnvoyProxyKubeProvider().EnvoyDeployment.Replicas = &repl - - deploy, err = kube.expectedProxyDeployment(infra) - require.NoError(t, err) - - // Check the number of replicas is as expected. - assert.Equal(t, repl, *deploy.Spec.Replicas) - - // Make sure no pod annotations are set by default - checkPodAnnotations(t, deploy, nil) -} - -func TestExpectedProxyDeployment(t *testing.T) { - testExpectedProxyDeployment(t, ir.NewInfra(), egcfgv1a1.DefaultResourceRequirements(), nil, nil) -} - -func TestExpectedProxyDeploymentForSpecifiedResources(t *testing.T) { - infra := ir.NewInfra() - requirements := corev1.ResourceRequirements{ - Limits: nil, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("128Mi")}, - Claims: nil, - } - containerSecurityContext := corev1.SecurityContext{ - RunAsUser: pointer.Int64(2000), - AllowPrivilegeEscalation: pointer.Bool(false), - } - FSGroupChangePolicy := func(s corev1.PodFSGroupChangePolicy) *corev1.PodFSGroupChangePolicy { return &s } - podSecurityContext := corev1.PodSecurityContext{ - RunAsUser: pointer.Int64(1000), - RunAsGroup: pointer.Int64(3000), - FSGroup: pointer.Int64(2000), - FSGroupChangePolicy: FSGroupChangePolicy(corev1.FSGroupChangeOnRootMismatch), - } - infra.Proxy.Config = &egcfgv1a1.EnvoyProxy{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: egcfgv1a1.EnvoyProxySpec{Provider: &egcfgv1a1.EnvoyProxyProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyProxyKubernetesProvider{ - EnvoyDeployment: &egcfgv1a1.KubernetesDeploymentSpec{ - Pod: &egcfgv1a1.KubernetesPodSpec{ - SecurityContext: &podSecurityContext, - }, - Container: &egcfgv1a1.KubernetesContainerSpec{ - Resources: &requirements, - SecurityContext: &containerSecurityContext, - }, - }, - }, - }}, - Status: egcfgv1a1.EnvoyProxyStatus{}, - } - - testExpectedProxyDeployment(t, infra, &requirements, &podSecurityContext, &containerSecurityContext) -} - -func TestExpectedBootstrap(t *testing.T) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - infra := ir.NewInfra() - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - // Set a custom bootstrap config into EnvoyProxy API and ensure the same - // value is set as the container arg. - bstrap := "blah" - infra.Proxy.Config = &egcfgv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "test", - }, - Spec: egcfgv1a1.EnvoyProxySpec{ - Bootstrap: &bstrap, - }, - } - - deploy, err := kube.expectedProxyDeployment(infra) - require.NoError(t, err) - container := checkContainer(t, deploy, envoyContainerName, true) - checkContainerHasArg(t, container, fmt.Sprintf("--config-yaml %s", bstrap)) -} - -func TestExpectedPodAnnotations(t *testing.T) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - infra := ir.NewInfra() - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - // Set service annotations into EnvoyProxy API and ensure the same - // value is set in the generated service. - annotations := map[string]string{ - "key1": "val1", - "key2": "val2", - } - infra.Proxy.Config = &egcfgv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "test", - }, - Spec: egcfgv1a1.EnvoyProxySpec{ - Provider: &egcfgv1a1.EnvoyProxyProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyProxyKubernetesProvider{ - EnvoyDeployment: &egcfgv1a1.KubernetesDeploymentSpec{ - Pod: &egcfgv1a1.KubernetesPodSpec{ - Annotations: annotations, - }, - }, - }, - }, - }, - } - - deploy, err := kube.expectedProxyDeployment(infra) - require.NoError(t, err) - checkPodAnnotations(t, deploy, annotations) -} - -func TestExpectedContainerPort(t *testing.T) { - const FooContainerPort, BarContainerPort = 7878, 8989 - - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - infra := ir.NewInfra() - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - infra.Proxy.Listeners = []ir.ProxyListener{ - { - Ports: []ir.ListenerPort{ - { - Name: "FooPort", - Protocol: ir.TCPProtocolType, - ContainerPort: FooContainerPort, - }, - }, - }, - { - Ports: []ir.ListenerPort{ - { - Name: "BarPort", - Protocol: ir.UDPProtocolType, - ContainerPort: BarContainerPort, - }, - }, - }, - } - - deploy, err := kube.expectedProxyDeployment(infra) - require.NoError(t, err) - ports := []int32{FooContainerPort, BarContainerPort} - for _, port := range ports { - checkContainerHasPort(t, deploy, port) - } -} +const ( + // envoyContainerName is the name of the Envoy container. + envoyContainerName = "envoy" +) func deploymentWithImage(deploy *appsv1.Deployment, image string) *appsv1.Deployment { dCopy := deploy.DeepCopy() @@ -267,13 +44,12 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) { cfg, err := config.New() require.NoError(t, err) - kube := NewInfra(nil, cfg) infra := ir.NewInfra() - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - deploy, err := kube.expectedProxyDeployment(infra) + r := proxy.NewResourceRender(cfg.Namespace, infra) + deploy, err := r.Deployment() require.NoError(t, err) testCases := []struct { @@ -329,18 +105,22 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateProxyDeployment(context.Background(), tc.in) + + kube := NewInfra(cli, cfg) + r := proxy.NewResourceRender(kube.Namespace, tc.in) + err := kube.createOrUpdateDeployment(context.Background(), r) require.NoError(t, err) actual := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) @@ -350,6 +130,10 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) { } func TestDeleteProxyDeployment(t *testing.T) { + cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() + cfg, err := config.New() + require.NoError(t, err) + testCases := []struct { name string expect bool @@ -363,19 +147,17 @@ func TestDeleteProxyDeployment(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - } - infra := ir.NewInfra() + kube := NewInfra(cli, cfg) + infra := ir.NewInfra() infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name + r := proxy.NewResourceRender(kube.Namespace, infra) - err := kube.createOrUpdateProxyDeployment(context.Background(), infra) + err := kube.createOrUpdateDeployment(context.Background(), r) require.NoError(t, err) - err = kube.deleteProxyDeployment(context.Background(), infra) + err = kube.deleteDeployment(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/proxy_infra.go b/internal/infrastructure/kubernetes/proxy_infra.go index a7dcdd0da5d..863d6368d3c 100644 --- a/internal/infrastructure/kubernetes/proxy_infra.go +++ b/internal/infrastructure/kubernetes/proxy_infra.go @@ -9,6 +9,7 @@ import ( "context" "errors" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" "github.com/envoyproxy/gateway/internal/ir" ) @@ -22,23 +23,8 @@ func (i *Infra) CreateOrUpdateProxyInfra(ctx context.Context, infra *ir.Infra) e return errors.New("infra proxy ir is nil") } - if err := i.createOrUpdateProxyServiceAccount(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateProxyConfigMap(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateProxyDeployment(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateProxyService(ctx, infra); err != nil { - return err - } - - return nil + r := proxy.NewResourceRender(i.Namespace, infra) + return i.createOrUpdate(ctx, r) } // DeleteProxyInfra removes the managed kube infra, if it doesn't exist. @@ -47,21 +33,6 @@ func (i *Infra) DeleteProxyInfra(ctx context.Context, infra *ir.Infra) error { return errors.New("infra ir is nil") } - if err := i.deleteProxyService(ctx, infra); err != nil { - return err - } - - if err := i.deleteProxyDeployment(ctx, infra); err != nil { - return err - } - - if err := i.deleteProxyConfigMap(ctx, infra); err != nil { - return err - } - - if err := i.deleteProxyServiceAccount(ctx, infra); err != nil { - return err - } - - return nil + r := proxy.NewResourceRender(i.Namespace, infra) + return i.delete(ctx, r) } diff --git a/internal/infrastructure/kubernetes/proxy_infra_test.go b/internal/infrastructure/kubernetes/proxy_infra_test.go index d37ac084c03..26158e27bde 100644 --- a/internal/infrastructure/kubernetes/proxy_infra_test.go +++ b/internal/infrastructure/kubernetes/proxy_infra_test.go @@ -16,15 +16,42 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) +func newTestInfra(t *testing.T) *Infra { + cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + return newTestInfraWithClient(t, cli) +} + +func newTestInfraWithClient(t *testing.T, cli client.Client) *Infra { + cfg, err := config.New() + require.NoError(t, err) + + cfg.EnvoyGateway = &egcfgv1a1.EnvoyGateway{ + TypeMeta: metav1.TypeMeta{}, + EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ + RateLimit: &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, + }, + }, + }, + } + + return NewInfra(cli, cfg) +} + func TestCreateProxyInfra(t *testing.T) { // Infra with Gateway owner labels. infraWithLabels := ir.NewInfra() - infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels = envoyAppLabel() + infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels = utils.EnvoyAppLabel() infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = "test-gw" @@ -61,10 +88,7 @@ func TestCreateProxyInfra(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "default", - } + kube := newTestInfra(t) // Create or update the proxy infra. err := kube.CreateOrUpdateProxyInfra(context.Background(), tc.in) if !tc.expect { @@ -76,7 +100,7 @@ func TestCreateProxyInfra(t *testing.T) { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(sa), sa)) @@ -84,7 +108,7 @@ func TestCreateProxyInfra(t *testing.T) { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(cm), cm)) @@ -92,7 +116,7 @@ func TestCreateProxyInfra(t *testing.T) { deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(deploy), deploy)) @@ -100,7 +124,7 @@ func TestCreateProxyInfra(t *testing.T) { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(svc), svc)) @@ -110,6 +134,7 @@ func TestCreateProxyInfra(t *testing.T) { } func TestDeleteProxyInfra(t *testing.T) { + testCases := []struct { name string in *ir.Infra @@ -131,9 +156,8 @@ func TestDeleteProxyInfra(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - } + kube := newTestInfra(t) + err := kube.DeleteProxyInfra(context.Background(), tc.in) if !tc.expect { require.Error(t, err) diff --git a/internal/infrastructure/kubernetes/proxy_labels.go b/internal/infrastructure/kubernetes/proxy_labels.go deleted file mode 100644 index c06a2952441..00000000000 --- a/internal/infrastructure/kubernetes/proxy_labels.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -// envoyAppLabel returns the labels used for all Envoy resources. -func envoyAppLabel() map[string]string { - return map[string]string{ - "app.gateway.envoyproxy.io/name": "envoy", - } -} - -// envoyLabels returns the labels, including extraLbls, used for Envoy resources. -func envoyLabels(extraLbls map[string]string) map[string]string { - lbls := envoyAppLabel() - for k, v := range extraLbls { - lbls[k] = v - } - - return lbls -} diff --git a/internal/infrastructure/kubernetes/proxy_labels_test.go b/internal/infrastructure/kubernetes/proxy_labels_test.go deleted file mode 100644 index 8fcc3902317..00000000000 --- a/internal/infrastructure/kubernetes/proxy_labels_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestEnvoyPodSelector(t *testing.T) { - cases := []struct { - name string - in map[string]string - expected map[string]string - }{ - { - name: "default", - in: map[string]string{"foo": "bar"}, - expected: map[string]string{ - "foo": "bar", - "app.gateway.envoyproxy.io/name": "envoy", - }, - }, - } - - for _, tc := range cases { - tc := tc - t.Run("", func(t *testing.T) { - got := getSelector(envoyLabels(tc.in)) - require.Equal(t, tc.expected, got.MatchLabels) - }) - } -} diff --git a/internal/infrastructure/kubernetes/proxy_service.go b/internal/infrastructure/kubernetes/proxy_service.go deleted file mode 100644 index f71d32955ac..00000000000 --- a/internal/infrastructure/kubernetes/proxy_service.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedProxyService returns the expected Service based on the provided infra. -func (i *Infra) expectedProxyService(infra *ir.Infra) (*corev1.Service, error) { - var ports []corev1.ServicePort - for _, listener := range infra.Proxy.Listeners { - for _, port := range listener.Ports { - target := intstr.IntOrString{IntVal: port.ContainerPort} - protocol := corev1.ProtocolTCP - if port.Protocol == ir.UDPProtocolType { - protocol = corev1.ProtocolUDP - } - p := corev1.ServicePort{ - Name: port.Name, - Protocol: protocol, - Port: port.ServicePort, - TargetPort: target, - } - ports = append(ports, p) - } - } - - // Set the labels based on the owning gatewayclass name. - labels := envoyLabels(infra.GetProxyInfra().GetProxyMetadata().Labels) - if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { - return nil, fmt.Errorf("missing owning gateway labels") - } - - // Get annotations - var annotations map[string]string - provider := infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider() - envoyServiceConfig := provider.GetEnvoyProxyKubeProvider().EnvoyService - if envoyServiceConfig.Annotations != nil { - annotations = envoyServiceConfig.Annotations - } - serviceSpec := expectedServiceSpec(envoyServiceConfig.Type) - serviceSpec.Ports = ports - serviceSpec.Selector = getSelector(labels).MatchLabels - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - Labels: labels, - Annotations: annotations, - }, - Spec: serviceSpec, - } - - return svc, nil -} - -// createOrUpdateProxyService creates a Service in the kube api server based on the provided infra, -// if it doesn't exist or updates it if it does. -func (i *Infra) createOrUpdateProxyService(ctx context.Context, infra *ir.Infra) error { - svc, err := i.expectedProxyService(infra) - if err != nil { - return fmt.Errorf("failed to generate expected service: %w", err) - } - return i.createOrUpdateService(ctx, svc) -} - -// deleteProxyService deletes the Envoy Service in the kube api server, if it exists. -func (i *Infra) deleteProxyService(ctx context.Context, infra *ir.Infra) error { - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - }, - } - - return i.deleteService(ctx, svc) -} diff --git a/internal/infrastructure/kubernetes/proxy_service_test.go b/internal/infrastructure/kubernetes/proxy_service_test.go index fbce8d6956f..6b3197a7882 100644 --- a/internal/infrastructure/kubernetes/proxy_service_test.go +++ b/internal/infrastructure/kubernetes/proxy_service_test.go @@ -9,129 +9,13 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway" - "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" "github.com/envoyproxy/gateway/internal/ir" ) -func testDesiredProxyService(t *testing.T, infra *ir.Infra, expected corev1.ServiceSpec) { - cfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, cfg) - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - infra.Proxy.Listeners[0].Ports = []ir.ListenerPort{ - { - Name: "gateway-system-gateway-1", - Protocol: ir.HTTPProtocolType, - ServicePort: 80, - ContainerPort: 2080, - }, - { - Name: "gateway-system-gateway-1", - Protocol: ir.HTTPSProtocolType, - ServicePort: 443, - ContainerPort: 2443, - }, - } - svc, err := kube.expectedProxyService(infra) - require.NoError(t, err) - - // Check the service name is as expected. - assert.Equal(t, svc.Name, expectedResourceHashedName(infra.Proxy.Name)) - - checkServiceHasPort(t, svc, 80) - checkServiceHasPort(t, svc, 443) - checkServiceHasTargetPort(t, svc, 2080) - checkServiceHasTargetPort(t, svc, 2443) - - // Ensure the Envoy service has the expected labels. - lbls := envoyAppLabel() - lbls[gatewayapi.OwningGatewayNamespaceLabel] = "default" - lbls[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - checkServiceHasLabels(t, svc, lbls) - - for _, port := range infra.Proxy.Listeners[0].Ports { - checkServiceHasPortName(t, svc, port.Name) - } - - // Make sure no service annotations are set by default - checkServiceHasAnnotations(t, svc, nil) - - // Make sure service type are set by default with ServiceTypeLoadBalancer - checkServiceSpec(t, svc, expected) -} - -func TestDesiredProxyService(t *testing.T) { - testDesiredProxyService(t, ir.NewInfra(), expectedServiceSpec(egcfgv1a1.DefaultKubernetesServiceType())) -} - -func TestDesiredProxySpecifiedServiceSpec(t *testing.T) { - infra := ir.NewInfra() - clusterIPServiceType := egcfgv1a1.GetKubernetesServiceType(egcfgv1a1.ServiceTypeClusterIP) - infra.Proxy.Config = &egcfgv1a1.EnvoyProxy{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: egcfgv1a1.EnvoyProxySpec{Provider: &egcfgv1a1.EnvoyProxyProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyProxyKubernetesProvider{ - EnvoyService: &egcfgv1a1.KubernetesServiceSpec{ - Type: clusterIPServiceType, - }, - }, - }}, - Status: egcfgv1a1.EnvoyProxyStatus{}, - } - testDesiredProxyService(t, infra, expectedServiceSpec(clusterIPServiceType)) -} - -func TestExpectedAnnotations(t *testing.T) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - infra := ir.NewInfra() - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - // Set service annotations into EnvoyProxy API and ensure the same - // value is set in the generated service. - annotations := map[string]string{ - "key1": "val1", - "key2": "val2", - } - infra.Proxy.Config = &egcfgv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "test", - }, - Spec: egcfgv1a1.EnvoyProxySpec{ - Provider: &egcfgv1a1.EnvoyProxyProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyProxyKubernetesProvider{ - EnvoyService: &egcfgv1a1.KubernetesServiceSpec{ - Annotations: annotations, - }, - }, - }, - }, - } - - svc, err := kube.expectedProxyService(infra) - require.NoError(t, err) - checkServiceHasAnnotations(t, svc, annotations) -} - func TestDeleteProxyService(t *testing.T) { testCases := []struct { name string @@ -144,19 +28,16 @@ func TestDeleteProxyService(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - } + kube := newTestInfra(t) infra := ir.NewInfra() infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - err := kube.createOrUpdateProxyService(context.Background(), infra) + r := proxy.NewResourceRender(kube.Namespace, infra) + err := kube.createOrUpdateService(context.Background(), r) require.NoError(t, err) - err = kube.deleteProxyService(context.Background(), infra) + err = kube.deleteService(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/proxy_serviceaccount.go b/internal/infrastructure/kubernetes/proxy_serviceaccount.go deleted file mode 100644 index 925be5564a9..00000000000 --- a/internal/infrastructure/kubernetes/proxy_serviceaccount.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedProxyServiceAccount returns the expected proxy serviceAccount. -func (i *Infra) expectedProxyServiceAccount(infra *ir.Infra) (*corev1.ServiceAccount, error) { - // Set the labels based on the owning gateway name. - labels := envoyLabels(infra.GetProxyInfra().GetProxyMetadata().Labels) - if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { - return nil, fmt.Errorf("missing owning gateway labels") - } - - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - Kind: "ServiceAccount", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - Labels: labels, - }, - }, nil -} - -// createOrUpdateProxyServiceAccount creates the Envoy ServiceAccount in the kube api server, -// if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateProxyServiceAccount(ctx context.Context, infra *ir.Infra) error { - sa, err := i.expectedProxyServiceAccount(infra) - if err != nil { - return err - } - return i.createOrUpdateServiceAccount(ctx, sa) -} - -// deleteProxyServiceAccount deletes the Envoy ServiceAccount in the kube api server, -// if it exists. -func (i *Infra) deleteProxyServiceAccount(ctx context.Context, infra *ir.Infra) error { - sa := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: expectedResourceHashedName(infra.Proxy.Name), - }, - } - - return i.deleteServiceAccount(ctx, sa) -} diff --git a/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go b/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go index eae2333275b..e0e9c208d73 100644 --- a/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go +++ b/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -22,36 +21,11 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) -func TestExpectedProxyServiceAccount(t *testing.T) { - cfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, cfg) - infra := ir.NewInfra() - - // An infra without Gateway owner labels should trigger - // an error. - _, err = kube.expectedProxyServiceAccount(infra) - require.NotNil(t, err) - - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - - sa, err := kube.expectedProxyServiceAccount(infra) - require.NoError(t, err) - - // Check the serviceaccount name is as expected. - assert.Equal(t, sa.Name, expectedResourceHashedName(infra.Proxy.Name)) - - wantLabels := envoyAppLabel() - wantLabels[gatewayapi.OwningGatewayNamespaceLabel] = "default" - wantLabels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name - assert.True(t, apiequality.Semantic.DeepEqual(wantLabels, sa.Labels)) -} - func TestCreateOrUpdateProxyServiceAccount(t *testing.T) { testCases := []struct { name string @@ -185,21 +159,27 @@ func TestCreateOrUpdateProxyServiceAccount(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Namespace: tc.ns, - } + cfg, err := config.New() + require.NoError(t, err) + cfg.Namespace = tc.ns + + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateProxyServiceAccount(context.Background(), tc.in) + + kube := NewInfra(cli, cfg) + + r := proxy.NewResourceRender(kube.Namespace, tc.in) + err = kube.createOrUpdateServiceAccount(context.Background(), r) require.NoError(t, err) actual := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: expectedResourceHashedName(tc.in.Proxy.Name), + Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) @@ -220,19 +200,17 @@ func TestDeleteProxyServiceAccount(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - } - infra := ir.NewInfra() + kube := newTestInfra(t) + infra := ir.NewInfra() infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name + r := proxy.NewResourceRender(kube.Namespace, infra) - err := kube.createOrUpdateProxyServiceAccount(context.Background(), infra) + err := kube.createOrUpdateServiceAccount(context.Background(), r) require.NoError(t, err) - err = kube.deleteProxyServiceAccount(context.Background(), infra) + err = kube.deleteServiceAccount(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/ratelimit/deployment.go b/internal/infrastructure/kubernetes/ratelimit/deployment.go new file mode 100644 index 00000000000..9eb31794fa2 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/deployment.go @@ -0,0 +1,168 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package ratelimit + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" +) + +const ( + // RedisSocketTypeEnvVar is the redis socket type. + RedisSocketTypeEnvVar = "REDIS_SOCKET_TYPE" + // RedisURLEnvVar is the redis url. + RedisURLEnvVar = "REDIS_URL" + // RuntimeRootEnvVar is the runtime root. + RuntimeRootEnvVar = "RUNTIME_ROOT" + // RuntimeSubdirectoryEnvVar is the runtime subdirectory. + RuntimeSubdirectoryEnvVar = "RUNTIME_SUBDIRECTORY" + // RuntimeIgnoreDotfilesEnvVar is the runtime ignoredotfiles. + RuntimeIgnoreDotfilesEnvVar = "RUNTIME_IGNOREDOTFILES" + // RuntimeWatchRootEnvVar is the runtime watch root. + RuntimeWatchRootEnvVar = "RUNTIME_WATCH_ROOT" + // LogLevelEnvVar is the log level. + LogLevelEnvVar = "LOG_LEVEL" + // UseStatsdEnvVar is the use statsd. + UseStatsdEnvVar = "USE_STATSD" + // InfraName is the name for rate-limit resources. + InfraName = "envoy-ratelimit" + // InfraGRPCPort is the grpc port that the rate limit service listens on. + InfraGRPCPort = 8081 +) + +// Deployment returns the expected rate limit Deployment based on the provided infra. +func (i *ResourceRender) Deployment() (*appsv1.Deployment, error) { + containers := expectedRateLimitContainers(i.ratelimit, i.rateLimitDeployment) + labels := rateLimitLabels() + selector := utils.GetSelector(labels) + + var annos map[string]string + if i.rateLimitDeployment.Pod.Annotations != nil { + annos = i.rateLimitDeployment.Pod.Annotations + } + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: InfraName, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: i.rateLimitDeployment.Replicas, + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selector.MatchLabels, + Annotations: annos, + }, + Spec: corev1.PodSpec{ + Containers: containers, + ServiceAccountName: InfraName, + AutomountServiceAccountToken: pointer.Bool(false), + TerminationGracePeriodSeconds: pointer.Int64(int64(300)), + DNSPolicy: corev1.DNSClusterFirst, + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + SecurityContext: i.rateLimitDeployment.Pod.SecurityContext, + Volumes: []corev1.Volume{ + { + Name: InfraName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: InfraName, + }, + DefaultMode: pointer.Int32(int32(420)), + Optional: pointer.Bool(false), + }, + }, + }, + }, + }, + }, + }, + } + + return deployment, nil +} + +func expectedRateLimitContainers(ratelimit *egcfgv1a1.RateLimit, rateLimitDeployment *egcfgv1a1.KubernetesDeploymentSpec) []corev1.Container { + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: InfraGRPCPort, + Protocol: corev1.ProtocolTCP, + }, + } + + containers := []corev1.Container{ + { + Name: InfraName, + Image: *rateLimitDeployment.Container.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "/bin/ratelimit", + }, + Env: []corev1.EnvVar{ + { + Name: RedisSocketTypeEnvVar, + Value: "tcp", + }, + { + Name: RedisURLEnvVar, + Value: ratelimit.Backend.Redis.URL, + }, + { + Name: RuntimeRootEnvVar, + Value: "/data", + }, + { + Name: RuntimeSubdirectoryEnvVar, + Value: "ratelimit", + }, + { + Name: RuntimeIgnoreDotfilesEnvVar, + Value: "true", + }, + { + Name: RuntimeWatchRootEnvVar, + Value: "false", + }, + { + Name: LogLevelEnvVar, + Value: "info", + }, + { + Name: UseStatsdEnvVar, + Value: "false", + }, + }, + Ports: ports, + Resources: *rateLimitDeployment.Container.Resources, + SecurityContext: rateLimitDeployment.Container.SecurityContext, + VolumeMounts: []corev1.VolumeMount{ + { + Name: InfraName, + MountPath: "/data/ratelimit/config", + ReadOnly: true, + }, + }, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + TerminationMessagePath: "/dev/termination-log", + }, + } + + return containers +} diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go new file mode 100644 index 00000000000..1caf96b4b8c --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -0,0 +1,116 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package ratelimit + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" + "github.com/envoyproxy/gateway/internal/ir" +) + +type ResourceRender struct { + // Namespace is the Namespace used for managed infra. + Namespace string + + infra *ir.RateLimitInfra + ratelimit *egcfgv1a1.RateLimit + rateLimitDeployment *egcfgv1a1.KubernetesDeploymentSpec +} + +// NewResourceRender returns a new ResourceRender. +func NewResourceRender(ns string, infra *ir.RateLimitInfra, rl *egcfgv1a1.RateLimit, deploy *egcfgv1a1.KubernetesDeploymentSpec) *ResourceRender { + return &ResourceRender{ + Namespace: ns, + infra: infra, + ratelimit: rl, + rateLimitDeployment: deploy, + } +} + +func (r *ResourceRender) Name() string { + return InfraName +} + +// ConfigMap returns the expected ConfigMap based on the provided infra. +func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { + labels := rateLimitLabels() + data := make(map[string]string) + + for _, config := range r.infra.ServiceConfigs { + data[config.Name] = config.Config + } + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: InfraName, + Labels: labels, + }, + Data: data, + }, nil +} + +// Service returns the expected rate limit Service based on the provided infra. +func (r *ResourceRender) Service() (*corev1.Service, error) { + ports := []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: InfraGRPCPort, + TargetPort: intstr.IntOrString{IntVal: InfraGRPCPort}, + }, + } + + labels := rateLimitLabels() + + serviceSpec := utils.ExpectedServiceSpec(egcfgv1a1.DefaultKubernetesServiceType()) + serviceSpec.Ports = ports + serviceSpec.Selector = utils.GetSelector(labels).MatchLabels + + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: InfraName, + Labels: labels, + }, + Spec: serviceSpec, + } + + return svc, nil +} + +// ServiceAccount returns the expected ratelimit serviceAccount. +func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: InfraName, + }, + }, nil +} + +// rateLimitLabels returns the labels used for all envoy rate limit resources. +func rateLimitLabels() map[string]string { + return map[string]string{ + "app.gateway.envoyproxy.io/name": InfraName, + } +} diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go new file mode 100644 index 00000000000..04874038020 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -0,0 +1,269 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package ratelimit + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/pointer" + "sigs.k8s.io/yaml" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/ir" +) + +var ( + rateLimitListener = "ratelimit-listener" + rateLimitConfig = ` +domain: first-listener +descriptors: + - key: first-route-key-rule-0-match-0 + value: first-route-value-rule-0-match-0 + rate_limit: + requests_per_unit: 5 + unit: second + unlimited: false + name: "" + replaces: [] + descriptors: [] + shadow_mode: false +` +) + +func TestRateLimitLabels(t *testing.T) { + cases := []struct { + name string + expected map[string]string + }{ + { + name: "ratelimit-labels", + expected: map[string]string{ + "app.gateway.envoyproxy.io/name": InfraName, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + got := rateLimitLabels() + require.Equal(t, tc.expected, got) + }) + } +} + +func TestServiceAccount(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + rateLimitInfra := new(ir.RateLimitInfra) + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + r := NewResourceRender(cfg.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + + sa, err := r.ServiceAccount() + require.NoError(t, err) + + expected, err := loadServiceAccount() + require.NoError(t, err) + + assert.Equal(t, expected, sa) +} + +func loadServiceAccount() (*corev1.ServiceAccount, error) { + saYAML, err := os.ReadFile("testdata/envoy-ratelimit-serviceaccount.yaml") + if err != nil { + return nil, err + } + sa := &corev1.ServiceAccount{} + _ = yaml.Unmarshal(saYAML, sa) + return sa, nil +} + +func TestConfigMap(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + rateLimitInfra := &ir.RateLimitInfra{ + ServiceConfigs: []*ir.RateLimitServiceConfig{ + { + Name: rateLimitListener, + Config: rateLimitConfig, + }, + }, + } + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + + r := NewResourceRender(cfg.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + cm, err := r.ConfigMap() + require.NoError(t, err) + + expected, err := loadConfigmap() + require.NoError(t, err) + + assert.Equal(t, expected, cm) +} + +func loadConfigmap() (*corev1.ConfigMap, error) { + cmYAML, err := os.ReadFile("testdata/envoy-ratelimit-configmap.yaml") + if err != nil { + return nil, err + } + cm := &corev1.ConfigMap{} + _ = yaml.Unmarshal(cmYAML, cm) + return cm, nil +} + +func TestService(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + rateLimitInfra := &ir.RateLimitInfra{ + ServiceConfigs: []*ir.RateLimitServiceConfig{ + { + Name: rateLimitListener, + Config: rateLimitConfig, + }, + }, + } + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + r := NewResourceRender(cfg.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + svc, err := r.Service() + require.NoError(t, err) + + expected, err := loadService() + require.NoError(t, err) + + assert.Equal(t, expected, svc) +} + +func loadService() (*corev1.Service, error) { + serviceYAML, err := os.ReadFile("testdata/envoy-ratelimit-service.yaml") + if err != nil { + return nil, err + } + svc := &corev1.Service{} + _ = yaml.Unmarshal(serviceYAML, svc) + return svc, nil +} + +func TestDeployment(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + deploy *egcfgv1a1.KubernetesDeploymentSpec + }{ + { + caseName: "default", + deploy: cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment, + }, + { + caseName: "custom", + deploy: &egcfgv1a1.KubernetesDeploymentSpec{ + Replicas: pointer.Int32(2), + Pod: &egcfgv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64(1000), + }, + }, + Container: &egcfgv1a1.KubernetesContainerSpec{ + Image: pointer.String("custom-image"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.Bool(true), + }, + }, + }, + }, + } + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + rateLimitInfra := &ir.RateLimitInfra{ + ServiceConfigs: []*ir.RateLimitServiceConfig{ + { + Name: rateLimitListener, + Config: rateLimitConfig, + }, + }, + } + + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + + r := NewResourceRender(cfg.Namespace, rateLimitInfra, rl, tc.deploy) + dp, err := r.Deployment() + require.NoError(t, err) + + expected, err := loadDeployment(tc.caseName) + require.NoError(t, err) + + assert.Equal(t, expected, dp) + }) + } +} + +func loadDeployment(caseName string) (*appsv1.Deployment, error) { + deploymentYAML, err := os.ReadFile(fmt.Sprintf("testdata/deployments/%s.yaml", caseName)) + if err != nil { + return nil, err + } + deployment := &appsv1.Deployment{} + _ = yaml.Unmarshal(deploymentYAML, deployment) + return deployment, nil +} + +func TestGetServiceURL(t *testing.T) { + got := GetServiceURL("envoy-gateway-system") + assert.Equal(t, "grpc://envoy-ratelimit.envoy-gateway-system.svc.cluster.local:8081", got) +} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml new file mode 100644 index 00000000000..7b32becb27e --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system +spec: + replicas: 2 + selector: + matchLabels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + template: + metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + annotations: + prometheus.io/scrape: "true" + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + image: custom-image + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: http + protocol: TCP + securityContext: + privileged: true + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/ratelimit/config + name: envoy-ratelimit + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + securityContext: + runAsUser: 1000 + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + defaultMode: 420 + name: envoy-ratelimit + optional: false + name: envoy-ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml new file mode 100644 index 00000000000..54243b0ea09 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system +spec: + replicas: 1 + selector: + matchLabels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + template: + metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: http + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/ratelimit/config + name: envoy-ratelimit + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + defaultMode: 420 + name: envoy-ratelimit + optional: false + name: envoy-ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-configmap.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-configmap.yaml new file mode 100644 index 00000000000..68219f2ab24 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system +data: + ratelimit-listener: | + + domain: first-listener + descriptors: + - key: first-route-key-rule-0-match-0 + value: first-route-value-rule-0-match-0 + rate_limit: + requests_per_unit: 5 + unit: second + unlimited: false + name: "" + replaces: [] + descriptors: [] + shadow_mode: false diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-service.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-service.yaml new file mode 100644 index 00000000000..9e12b26203c --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.gateway.envoyproxy.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system +spec: + type: LoadBalancer + sessionAffinity: None + ExternalTrafficPolicy: Local + ports: + - name: http + port: 8081 + protocol: TCP + targetPort: 8081 + selector: + app.gateway.envoyproxy.io/name: envoy-ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-serviceaccount.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-serviceaccount.yaml new file mode 100644 index 00000000000..b7ce205b345 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/envoy-ratelimit-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: envoy-ratelimit + namespace: envoy-gateway-system diff --git a/internal/infrastructure/kubernetes/ratelimit/utils.go b/internal/infrastructure/kubernetes/ratelimit/utils.go new file mode 100644 index 00000000000..9649bb0cff6 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/utils.go @@ -0,0 +1,14 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package ratelimit + +import "fmt" + +// GetServiceURL returns the URL for the rate limit service. +// TODO: support custom trust domain +func GetServiceURL(namespace string) string { + return fmt.Sprintf("grpc://%s.%s.svc.cluster.local:%d", InfraName, namespace, InfraGRPCPort) +} diff --git a/internal/infrastructure/kubernetes/ratelimit_configmap.go b/internal/infrastructure/kubernetes/ratelimit_configmap.go deleted file mode 100644 index cbb098654ae..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_configmap.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedRateLimitConfigMap returns the expected ConfigMap based on the provided infra. -func (i *Infra) expectedRateLimitConfigMap(infra *ir.RateLimitInfra) *corev1.ConfigMap { - labels := rateLimitLabels() - data := make(map[string]string) - - for _, serviceConfig := range infra.ServiceConfigs { - data[serviceConfig.Name] = serviceConfig.Config - } - - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - Labels: labels, - }, - Data: data, - } -} - -// createOrUpdateRateLimitConfigMap creates a ConfigMap in the Kube api server based on the provided -// infra, if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateRateLimitConfigMap(ctx context.Context, infra *ir.RateLimitInfra) error { - cm := i.expectedRateLimitConfigMap(infra) - return i.createOrUpdateConfigMap(ctx, cm) -} - -// deleteProxyConfigMap deletes the Envoy ConfigMap in the kube api server, if it exists. -func (i *Infra) deleteRateLimitConfigMap(ctx context.Context, _ *ir.RateLimitInfra) error { - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - }, - } - - return i.deleteConfigMap(ctx, cm) -} diff --git a/internal/infrastructure/kubernetes/ratelimit_configmap_test.go b/internal/infrastructure/kubernetes/ratelimit_configmap_test.go index 452ef5509eb..ac25f939f27 100644 --- a/internal/infrastructure/kubernetes/ratelimit_configmap_test.go +++ b/internal/infrastructure/kubernetes/ratelimit_configmap_test.go @@ -17,8 +17,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" ) @@ -40,49 +42,24 @@ descriptors: ` ) -func TestExpectedRateLimitConfigMap(t *testing.T) { - // Setup the ratelimit infra. - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - cfg, err := config.New() - require.NoError(t, err) - - kube := NewInfra(cli, cfg) - - rateLimitInfra := new(ir.RateLimitInfra) - - c := &ir.RateLimitServiceConfig{ - Name: rateLimitListener, - Config: rateLimitConfig, - } - rateLimitInfra.ServiceConfigs = append(rateLimitInfra.ServiceConfigs, c) - - // An infra without Gateway owner labels should trigger - // an error. - cm := kube.expectedRateLimitConfigMap(rateLimitInfra) - require.NotNil(t, cm) - - require.Equal(t, "envoy-ratelimit", cm.Name) - require.Equal(t, "envoy-gateway-system", cm.Namespace) - require.Contains(t, cm.Data, rateLimitListener) - assert.Equal(t, rateLimitConfig, cm.Data[rateLimitListener]) - - wantLabels := rateLimitLabels() - assert.True(t, apiequality.Semantic.DeepEqual(wantLabels, cm.Labels)) -} - func TestCreateOrUpdateRateLimitConfigMap(t *testing.T) { cfg, err := config.New() require.NoError(t, err) - kube := NewInfra(nil, cfg) - - cfg.Namespace = "envoy-gateway-system" rateLimitInfra := new(ir.RateLimitInfra) - c := &ir.RateLimitServiceConfig{ + rateLimitInfra.ServiceConfigs = append(rateLimitInfra.ServiceConfigs, &ir.RateLimitServiceConfig{ Name: rateLimitListener, Config: rateLimitConfig, + }) + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, } - rateLimitInfra.ServiceConfigs = append(rateLimitInfra.ServiceConfigs, c) + r := ratelimit.NewResourceRender(cfg.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) testCases := []struct { name string @@ -94,9 +71,9 @@ func TestCreateOrUpdateRateLimitConfigMap(t *testing.T) { expect: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: cfg.Namespace, - Name: rateLimitInfraName, + Name: ratelimit.InfraName, Labels: map[string]string{ - "app.gateway.envoyproxy.io/name": rateLimitInfraName, + "app.gateway.envoyproxy.io/name": ratelimit.InfraName, }, }, Data: map[string]string{rateLimitListener: rateLimitConfig}, @@ -107,9 +84,9 @@ func TestCreateOrUpdateRateLimitConfigMap(t *testing.T) { current: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: cfg.Namespace, - Name: rateLimitInfraName, + Name: ratelimit.InfraName, Labels: map[string]string{ - "app.gateway.envoyproxy.io/name": rateLimitInfraName, + "app.gateway.envoyproxy.io/name": ratelimit.InfraName, }, }, Data: map[string]string{"foo": "bar"}, @@ -117,9 +94,9 @@ func TestCreateOrUpdateRateLimitConfigMap(t *testing.T) { expect: &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: cfg.Namespace, - Name: rateLimitInfraName, + Name: ratelimit.InfraName, Labels: map[string]string{ - "app.gateway.envoyproxy.io/name": rateLimitInfraName, + "app.gateway.envoyproxy.io/name": ratelimit.InfraName, }, }, Data: map[string]string{rateLimitListener: rateLimitConfig}, @@ -130,12 +107,16 @@ func TestCreateOrUpdateRateLimitConfigMap(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateRateLimitConfigMap(context.Background(), rateLimitInfra) + + kube := NewInfra(cli, cfg) + + err := kube.createOrUpdateConfigMap(context.Background(), r) require.NoError(t, err) actual := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -154,7 +135,14 @@ func TestDeleteRateLimitConfigMap(t *testing.T) { cfg, err := config.New() require.NoError(t, err) - rateLimitInfra := new(ir.RateLimitInfra) + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } testCases := []struct { name string @@ -189,10 +177,13 @@ func TestDeleteRateLimitConfigMap(t *testing.T) { cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() kube := NewInfra(cli, cfg) - err := kube.createOrUpdateRateLimitConfigMap(context.Background(), rateLimitInfra) + rateLimitInfra := new(ir.RateLimitInfra) + r := ratelimit.NewResourceRender(kube.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + + err := kube.createOrUpdateConfigMap(context.Background(), r) require.NoError(t, err) - err = kube.deleteRateLimitConfigMap(context.Background(), rateLimitInfra) + err = kube.deleteConfigMap(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/ratelimit_deployment.go b/internal/infrastructure/kubernetes/ratelimit_deployment.go deleted file mode 100644 index 3bf2ea1d543..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_deployment.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - // Register embed - _ "embed" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - - egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" - "github.com/envoyproxy/gateway/internal/ir" -) - -const ( - // rateLimitRedisSocketTypeEnvVar is the redis socket type. - rateLimitRedisSocketTypeEnvVar = "REDIS_SOCKET_TYPE" - // rateLimitRedisURLEnvVar is the redis url. - rateLimitRedisURLEnvVar = "REDIS_URL" - // rateLimitRuntimeRootEnvVar is the runtime root. - rateLimitRuntimeRootEnvVar = "RUNTIME_ROOT" - // rateLimitRuntimeSubdirectoryEnvVar is the runtime subdirectory. - rateLimitRuntimeSubdirectoryEnvVar = "RUNTIME_SUBDIRECTORY" - // rateLimitRuntimeIgnoreDotfilesEnvVar is the runtime ignoredotfiles. - rateLimitRuntimeIgnoreDotfilesEnvVar = "RUNTIME_IGNOREDOTFILES" - // rateLimitRuntimeWatchRootEnvVar is the runtime watch root. - rateLimitRuntimeWatchRootEnvVar = "RUNTIME_WATCH_ROOT" - // rateLimitLogLevelEnvVar is the log level. - rateLimitLogLevelEnvVar = "LOG_LEVEL" - // rateLimitUseStatsdEnvVar is the use statsd. - rateLimitUseStatsdEnvVar = "USE_STATSD" - // rateLimitInfraName is the name for rate-limit resources. - rateLimitInfraName = "envoy-ratelimit" - // rateLimitInfraGRPCPort is the grpc port that the rate limit service listens on. - rateLimitInfraGRPCPort = 8081 -) - -// expectedRateLimitDeployment returns the expected rate limit Deployment based on the provided infra. -func (i *Infra) expectedRateLimitDeployment(infra *ir.RateLimitInfra) *appsv1.Deployment { - rateLimitDeployment := i.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment - containers := expectedRateLimitContainers(infra, i.EnvoyGateway.RateLimit, rateLimitDeployment) - labels := rateLimitLabels() - selector := getSelector(labels) - - // Get annotations - var annotations map[string]string - if rateLimitDeployment.Pod.Annotations != nil { - annotations = rateLimitDeployment.Pod.Annotations - } - - deployment := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: rateLimitDeployment.Replicas, - Selector: selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: selector.MatchLabels, - Annotations: annotations, - }, - Spec: corev1.PodSpec{ - Containers: containers, - ServiceAccountName: rateLimitInfraName, - AutomountServiceAccountToken: pointer.Bool(false), - TerminationGracePeriodSeconds: pointer.Int64(int64(300)), - DNSPolicy: corev1.DNSClusterFirst, - RestartPolicy: corev1.RestartPolicyAlways, - SchedulerName: "default-scheduler", - SecurityContext: rateLimitDeployment.Pod.SecurityContext, - Volumes: []corev1.Volume{ - { - Name: rateLimitInfraName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rateLimitInfraName, - }, - DefaultMode: pointer.Int32(int32(420)), - Optional: pointer.Bool(false), - }, - }, - }, - }, - }, - }, - }, - } - - return deployment -} - -func expectedRateLimitContainers(_ *ir.RateLimitInfra, rateLimit *egcfgv1a1.RateLimit, rateLimitDeployment *egcfgv1a1.KubernetesDeploymentSpec) []corev1.Container { - ports := []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: rateLimitInfraGRPCPort, - Protocol: corev1.ProtocolTCP, - }, - } - - containers := []corev1.Container{ - { - Name: rateLimitInfraName, - Image: *rateLimitDeployment.Container.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/ratelimit", - }, - Env: []corev1.EnvVar{ - { - Name: rateLimitRedisSocketTypeEnvVar, - Value: "tcp", - }, - { - Name: rateLimitRedisURLEnvVar, - Value: rateLimit.Backend.Redis.URL, - }, - { - Name: rateLimitRuntimeRootEnvVar, - Value: "/data", - }, - { - Name: rateLimitRuntimeSubdirectoryEnvVar, - Value: "ratelimit", - }, - { - Name: rateLimitRuntimeIgnoreDotfilesEnvVar, - Value: "true", - }, - { - Name: rateLimitRuntimeWatchRootEnvVar, - Value: "false", - }, - { - Name: rateLimitLogLevelEnvVar, - Value: "info", - }, - { - Name: rateLimitUseStatsdEnvVar, - Value: "false", - }, - }, - Ports: ports, - Resources: *rateLimitDeployment.Container.Resources, - SecurityContext: rateLimitDeployment.Container.SecurityContext, - VolumeMounts: []corev1.VolumeMount{ - { - Name: rateLimitInfraName, - MountPath: "/data/ratelimit/config", - ReadOnly: true, - }, - }, - TerminationMessagePolicy: corev1.TerminationMessageReadFile, - TerminationMessagePath: "/dev/termination-log", - }, - } - - return containers -} - -// createOrUpdateRateLimitDeployment creates a Deployment in the kube api server based on the provided -// infra, if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateRateLimitDeployment(ctx context.Context, infra *ir.RateLimitInfra) error { - deployment := i.expectedRateLimitDeployment(infra) - return i.createOrUpdateDeployment(ctx, deployment) -} - -// deleteRateLimitDeployment deletes the Envoy RateLimit Deployment in the kube api server, if it exists. -func (i *Infra) deleteRateLimitDeployment(ctx context.Context, _ *ir.RateLimitInfra) error { - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - }, - } - - return i.deleteDeployment(ctx, deployment) -} diff --git a/internal/infrastructure/kubernetes/ratelimit_deployment_test.go b/internal/infrastructure/kubernetes/ratelimit_deployment_test.go index 51e1140c58a..4f692d7bb96 100644 --- a/internal/infrastructure/kubernetes/ratelimit_deployment_test.go +++ b/internal/infrastructure/kubernetes/ratelimit_deployment_test.go @@ -9,11 +9,8 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -21,153 +18,27 @@ import ( egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" ) -func testExpectedRateLimitDeployment(t *testing.T, envoyGateway *egcfgv1a1.EnvoyGateway, rateLimitInfra *ir.RateLimitInfra, expected *corev1.ResourceRequirements) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - kube.EnvoyGateway = envoyGateway - - deployment := kube.expectedRateLimitDeployment(rateLimitInfra) - - // Check the deployment name is as expected. - assert.Equal(t, deployment.Name, rateLimitInfraName) - - // Check container details, i.e. env vars, labels, etc. for the deployment are as expected. - container := checkContainer(t, deployment, rateLimitInfraName, true) - checkContainerImage(t, container, egcfgv1a1.DefaultRateLimitImage) - checkContainerResources(t, container, expected) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRedisSocketTypeEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRedisURLEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRuntimeRootEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRuntimeSubdirectoryEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRuntimeIgnoreDotfilesEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitRuntimeWatchRootEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitLogLevelEnvVar) - checkEnvVar(t, deployment, rateLimitInfraName, rateLimitUseStatsdEnvVar) - checkLabels(t, deployment, deployment.Labels) - - // Check container ports for the deployment are as expected. - ports := []int32{rateLimitInfraGRPCPort} - for _, port := range ports { - checkContainerHasPort(t, deployment, port) - } - - // Set the deployment replicas. - repl := int32(1) - // Check the number of replicas is as expected. - assert.Equal(t, repl, *deployment.Spec.Replicas) - - // Make sure no pod annotations are set by default - checkPodAnnotations(t, deployment, nil) -} - -func TestExpectedRateLimitDeployment(t *testing.T) { - envoyGateway := &egcfgv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{}, - EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ - RateLimit: &egcfgv1a1.RateLimit{ - Backend: egcfgv1a1.RateLimitDatabaseBackend{ - Type: egcfgv1a1.RedisBackendType, - Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, - }, - }, - }, - } - rateLimitInfra := &ir.RateLimitInfra{} - testExpectedRateLimitDeployment(t, envoyGateway, rateLimitInfra, egcfgv1a1.DefaultResourceRequirements()) -} - -func TestExpectedRateLimitDeploymentForSpecifiedResources(t *testing.T) { - requirements := corev1.ResourceRequirements{ - Limits: nil, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("128Mi")}, - Claims: nil, - } - envoyGateway := &egcfgv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{}, - EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ - Provider: &egcfgv1a1.EnvoyGatewayProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyGatewayKubernetesProvider{RateLimitDeployment: &egcfgv1a1.KubernetesDeploymentSpec{ - Container: &egcfgv1a1.KubernetesContainerSpec{Resources: &requirements}}, - }, - }, - RateLimit: &egcfgv1a1.RateLimit{ - Backend: egcfgv1a1.RateLimitDatabaseBackend{ - Type: egcfgv1a1.RedisBackendType, - Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, - }, - }, - }, - } - rateLimitInfra := &ir.RateLimitInfra{} - testExpectedRateLimitDeployment(t, envoyGateway, rateLimitInfra, &requirements) -} - -func TestExpectedRateLimitPodAnnotations(t *testing.T) { - svrCfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, svrCfg) - - // Set service annotations into EnvoyProxy API and ensure the same - // value is set in the generated service. - annotations := map[string]string{ - "key1": "val1", - "key2": "val2", - } - - kube.EnvoyGateway = &egcfgv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{}, - EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ - Provider: &egcfgv1a1.EnvoyGatewayProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - Kubernetes: &egcfgv1a1.EnvoyGatewayKubernetesProvider{RateLimitDeployment: &egcfgv1a1.KubernetesDeploymentSpec{ - Pod: &egcfgv1a1.KubernetesPodSpec{Annotations: annotations}}, - }, - }, - RateLimit: &egcfgv1a1.RateLimit{ - Backend: egcfgv1a1.RateLimitDatabaseBackend{ - Type: egcfgv1a1.RedisBackendType, - Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, - }, - }, - }, - } - - rateLimitInfra := &ir.RateLimitInfra{} - - deploy := kube.expectedRateLimitDeployment(rateLimitInfra) - checkPodAnnotations(t, deploy, annotations) -} - func TestCreateOrUpdateRateLimitDeployment(t *testing.T) { cfg, err := config.New() require.NoError(t, err) - kube := NewInfra(nil, cfg) - kube.EnvoyGateway = &egcfgv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{}, - EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ - Provider: &egcfgv1a1.EnvoyGatewayProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - }, - RateLimit: &egcfgv1a1.RateLimit{ - Backend: egcfgv1a1.RateLimitDatabaseBackend{ - Type: egcfgv1a1.RedisBackendType, - Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, - }, + rateLimitInfra := new(ir.RateLimitInfra) + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", }, }, } - rateLimitInfra := &ir.RateLimitInfra{} - deployment := kube.expectedRateLimitDeployment(rateLimitInfra) + + r := ratelimit.NewResourceRender(cfg.Namespace, rateLimitInfra, rl, cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + deployment, err := r.Deployment() + require.NoError(t, err) testCases := []struct { name string @@ -197,18 +68,22 @@ func TestCreateOrUpdateRateLimitDeployment(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateRateLimitDeployment(context.Background(), tc.in) + + kube := NewInfra(cli, cfg) + r := ratelimit.NewResourceRender(kube.Namespace, tc.in, rl, kube.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + err := kube.createOrUpdateDeployment(context.Background(), r) require.NoError(t, err) actual := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: rateLimitInfraName, + Name: ratelimit.InfraName, }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) @@ -218,6 +93,15 @@ func TestCreateOrUpdateRateLimitDeployment(t *testing.T) { } func TestDeleteRateLimitDeployment(t *testing.T) { + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + testCases := []struct { name string expect bool @@ -231,29 +115,13 @@ func TestDeleteRateLimitDeployment(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - EnvoyGateway: &egcfgv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{}, - EnvoyGatewaySpec: egcfgv1a1.EnvoyGatewaySpec{ - Provider: &egcfgv1a1.EnvoyGatewayProvider{ - Type: egcfgv1a1.ProviderTypeKubernetes, - }, - RateLimit: &egcfgv1a1.RateLimit{ - Backend: egcfgv1a1.RateLimitDatabaseBackend{ - Type: egcfgv1a1.RedisBackendType, - Redis: &egcfgv1a1.RateLimitRedisSettings{URL: ""}, - }, - }, - }, - }, - } - rateLimitInfra := &ir.RateLimitInfra{} - err := kube.createOrUpdateRateLimitDeployment(context.Background(), rateLimitInfra) + kube := newTestInfra(t) + rateLimitInfra := new(ir.RateLimitInfra) + r := ratelimit.NewResourceRender(kube.Namespace, rateLimitInfra, rl, kube.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + err := kube.createOrUpdateDeployment(context.Background(), r) require.NoError(t, err) - err = kube.deleteRateLimitDeployment(context.Background(), rateLimitInfra) + err = kube.deleteDeployment(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/ratelimit_infra.go b/internal/infrastructure/kubernetes/ratelimit_infra.go index 8853659e2d9..9e5e0388cc4 100644 --- a/internal/infrastructure/kubernetes/ratelimit_infra.go +++ b/internal/infrastructure/kubernetes/ratelimit_infra.go @@ -9,6 +9,7 @@ import ( "context" "errors" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" ) @@ -17,24 +18,10 @@ func (i *Infra) CreateOrUpdateRateLimitInfra(ctx context.Context, infra *ir.Rate if infra == nil { return errors.New("ratelimit infra ir is nil") } - if err := i.createOrUpdateRateLimitServiceAccount(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateRateLimitConfigMap(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateRateLimitDeployment(ctx, infra); err != nil { - return err - } - - if err := i.createOrUpdateRateLimitService(ctx, infra); err != nil { - return err - } - return nil + r := ratelimit.NewResourceRender(i.Namespace, infra, i.EnvoyGateway.RateLimit, i.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + return i.createOrUpdate(ctx, r) } // DeleteRateLimitInfra removes the managed kube infra, if it doesn't exist. @@ -43,21 +30,6 @@ func (i *Infra) DeleteRateLimitInfra(ctx context.Context, infra *ir.RateLimitInf return errors.New("ratelimit infra ir is nil") } - if err := i.deleteRateLimitService(ctx, infra); err != nil { - return err - } - - if err := i.deleteRateLimitDeployment(ctx, infra); err != nil { - return err - } - - if err := i.deleteRateLimitConfigMap(ctx, infra); err != nil { - return err - } - - if err := i.deleteRateLimitServiceAccount(ctx, infra); err != nil { - return err - } - - return nil + r := ratelimit.NewResourceRender(i.Namespace, infra, i.EnvoyGateway.RateLimit, i.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + return i.delete(ctx, r) } diff --git a/internal/infrastructure/kubernetes/ratelimit_infra_test.go b/internal/infrastructure/kubernetes/ratelimit_infra_test.go new file mode 100644 index 00000000000..b1666e084e1 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit_infra_test.go @@ -0,0 +1,122 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package kubernetes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" + "github.com/envoyproxy/gateway/internal/ir" +) + +func TestCreateRateLimitInfra(t *testing.T) { + rateLimitInfra := new(ir.RateLimitInfra) + + testCases := []struct { + name string + in *ir.RateLimitInfra + expect bool + }{ + { + name: "nil-infra", + in: nil, + expect: false, + }, + { + name: "default infra", + in: rateLimitInfra, + expect: true, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + kube := newTestInfra(t) + err := kube.CreateOrUpdateRateLimitInfra(context.Background(), tc.in) + if !tc.expect { + require.Error(t, err) + } else { + require.NoError(t, err) + + // Verify all resources were created via the fake kube client. + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kube.Namespace, + Name: ratelimit.InfraName, + }, + } + require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(sa), sa)) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kube.Namespace, + Name: ratelimit.InfraName, + }, + } + require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(cm), cm)) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kube.Namespace, + Name: ratelimit.InfraName, + }, + } + require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(deploy), deploy)) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kube.Namespace, + Name: ratelimit.InfraName, + }, + } + require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(svc), svc)) + } + }) + } +} + +func TestDeleteRateLimitInfra(t *testing.T) { + testCases := []struct { + name string + in *ir.RateLimitInfra + expect bool + }{ + { + name: "nil infra", + in: nil, + expect: false, + }, + { + name: "default infra", + in: new(ir.RateLimitInfra), + expect: true, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + kube := newTestInfra(t) + + err := kube.DeleteRateLimitInfra(context.Background(), tc.in) + if !tc.expect { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/infrastructure/kubernetes/ratelimit_labels.go b/internal/infrastructure/kubernetes/ratelimit_labels.go deleted file mode 100644 index b6233f5e71e..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_labels.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -// rateLimitLabels returns the labels used for all envoy rate limit resources. -func rateLimitLabels() map[string]string { - return map[string]string{ - "app.gateway.envoyproxy.io/name": rateLimitInfraName, - } -} diff --git a/internal/infrastructure/kubernetes/ratelimit_labels_test.go b/internal/infrastructure/kubernetes/ratelimit_labels_test.go deleted file mode 100644 index cfd66251488..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_labels_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestRateLimitLabels(t *testing.T) { - cases := []struct { - name string - expected map[string]string - }{ - { - name: "ratelimit-labels", - expected: map[string]string{ - "app.gateway.envoyproxy.io/name": rateLimitInfraName, - }, - }, - } - - for _, tc := range cases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - got := rateLimitLabels() - require.Equal(t, tc.expected, got) - }) - } -} diff --git a/internal/infrastructure/kubernetes/ratelimit_service.go b/internal/infrastructure/kubernetes/ratelimit_service.go deleted file mode 100644 index 48362d5b927..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_service.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedRateLimitInfraService returns the expected rate limit Service based on the provided infra. -func (i *Infra) expectedRateLimitService(_ *ir.RateLimitInfra) *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - Port: rateLimitInfraGRPCPort, - TargetPort: intstr.IntOrString{IntVal: rateLimitInfraGRPCPort}, - }, - } - - labels := rateLimitLabels() - - serviceSpec := expectedServiceSpec(egcfgv1a1.DefaultKubernetesServiceType()) - serviceSpec.Ports = ports - serviceSpec.Selector = getSelector(labels).MatchLabels - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - Labels: labels, - }, - Spec: serviceSpec, - } - - return svc -} - -// GetRateLimitServiceURL returns the URL for the rate limit service. -func GetRateLimitServiceURL(namespace string) string { - return fmt.Sprintf("grpc://%s.%s.svc.cluster.local:%d", rateLimitInfraName, namespace, rateLimitInfraGRPCPort) -} - -// createOrUpdateRateLimitService creates a Service in the kube api server based on the provided infra, -// if it doesn't exist or updates it if it does. -func (i *Infra) createOrUpdateRateLimitService(ctx context.Context, infra *ir.RateLimitInfra) error { - svc := i.expectedRateLimitService(infra) - return i.createOrUpdateService(ctx, svc) -} - -// deleteRateLimitService deletes the rate limit Service in the kube api server, if it exists. -func (i *Infra) deleteRateLimitService(ctx context.Context, _ *ir.RateLimitInfra) error { - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - }, - } - - return i.deleteService(ctx, svc) -} diff --git a/internal/infrastructure/kubernetes/ratelimit_service_test.go b/internal/infrastructure/kubernetes/ratelimit_service_test.go index 0944d6a4aee..1de360deb7f 100644 --- a/internal/infrastructure/kubernetes/ratelimit_service_test.go +++ b/internal/infrastructure/kubernetes/ratelimit_service_test.go @@ -9,38 +9,23 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway" - "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" ) -func TestDesiredRateLimitService(t *testing.T) { - cfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, cfg) - rateLimitInfra := new(ir.RateLimitInfra) - svc := kube.expectedRateLimitService(rateLimitInfra) - - // Check the service name is as expected. - assert.Equal(t, svc.Name, rateLimitInfraName) - - checkServiceHasPort(t, svc, rateLimitInfraGRPCPort) - - // Ensure the Envoy RateLimit service has the expected labels. - lbls := rateLimitLabels() - checkServiceHasLabels(t, svc, lbls) - - // Make sure service type are set by default with ServiceTypeLoadBalancer - checkServiceSpec(t, svc, expectedServiceSpec(egcfgv1a1.DefaultKubernetesServiceType())) -} - func TestDeleteRateLimitService(t *testing.T) { + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + testCases := []struct { name string }{ @@ -52,16 +37,14 @@ func TestDeleteRateLimitService(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - } - rateLimitInfra := new(ir.RateLimitInfra) + kube := newTestInfra(t) - err := kube.createOrUpdateRateLimitService(context.Background(), rateLimitInfra) + rateLimitInfra := new(ir.RateLimitInfra) + r := ratelimit.NewResourceRender(kube.Namespace, rateLimitInfra, rl, kube.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + err := kube.createOrUpdateService(context.Background(), r) require.NoError(t, err) - err = kube.deleteRateLimitService(context.Background(), rateLimitInfra) + err = kube.deleteService(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/ratelimit_serviceaccount.go b/internal/infrastructure/kubernetes/ratelimit_serviceaccount.go deleted file mode 100644 index 39fbc3fc03b..00000000000 --- a/internal/infrastructure/kubernetes/ratelimit_serviceaccount.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/envoyproxy/gateway/internal/ir" -) - -// expectedRateLimitServiceAccount returns the expected ratelimit serviceAccount. -func (i *Infra) expectedRateLimitServiceAccount(_ *ir.RateLimitInfra) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - Kind: "ServiceAccount", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - }, - } -} - -// createOrUpdateRateLimitServiceAccount creates the Envoy RateLimit ServiceAccount in the kube api server, -// if it doesn't exist and updates it if it does. -func (i *Infra) createOrUpdateRateLimitServiceAccount(ctx context.Context, infra *ir.RateLimitInfra) error { - sa := i.expectedRateLimitServiceAccount(infra) - return i.createOrUpdateServiceAccount(ctx, sa) -} - -// deleteRateLimitServiceAccount deletes the Envoy RateLimit ServiceAccount in the kube api server, -// if it exists. -func (i *Infra) deleteRateLimitServiceAccount(ctx context.Context, _ *ir.RateLimitInfra) error { - sa := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: i.Namespace, - Name: rateLimitInfraName, - }, - } - - return i.deleteServiceAccount(ctx, sa) -} diff --git a/internal/infrastructure/kubernetes/ratelimit_serviceaccount_test.go b/internal/infrastructure/kubernetes/ratelimit_serviceaccount_test.go index 3c3a1cd77e9..ea856ba6af0 100644 --- a/internal/infrastructure/kubernetes/ratelimit_serviceaccount_test.go +++ b/internal/infrastructure/kubernetes/ratelimit_serviceaccount_test.go @@ -18,29 +18,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" ) -func TestExpectedRateLimitServiceAccount(t *testing.T) { - cfg, err := config.New() - require.NoError(t, err) - cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build() - kube := NewInfra(cli, cfg) - - rateLimitInfra := new(ir.RateLimitInfra) - - // An infra without Gateway owner labels should trigger - // an error. - sa := kube.expectedRateLimitServiceAccount(rateLimitInfra) - require.NotNil(t, sa) - - // Check the serviceaccount name is as expected. - assert.Equal(t, sa.Name, rateLimitInfraName) -} - func TestCreateOrUpdateRateLimitServiceAccount(t *testing.T) { + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + testCases := []struct { name string ns string @@ -59,7 +53,7 @@ func TestCreateOrUpdateRateLimitServiceAccount(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: "envoy-gateway-system", - Name: rateLimitInfraName, + Name: ratelimit.InfraName, }, }, }, @@ -74,7 +68,7 @@ func TestCreateOrUpdateRateLimitServiceAccount(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: "envoy-gateway-system", - Name: rateLimitInfraName, + Name: ratelimit.InfraName, }, }, }, @@ -83,21 +77,28 @@ func TestCreateOrUpdateRateLimitServiceAccount(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - kube := &Infra{ - Namespace: tc.ns, - } + var cli client.Client if tc.current != nil { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build() } else { - kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() + cli = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build() } - err := kube.createOrUpdateRateLimitServiceAccount(context.Background(), tc.in) + + cfg, err := config.New() + require.NoError(t, err) + cfg.Namespace = tc.ns + + kube := NewInfra(cli, cfg) + + r := ratelimit.NewResourceRender(kube.Namespace, tc.in, rl, kube.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + + err = kube.createOrUpdateServiceAccount(context.Background(), r) require.NoError(t, err) actual := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: rateLimitInfraName, + Name: ratelimit.InfraName, }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) @@ -109,6 +110,15 @@ func TestCreateOrUpdateRateLimitServiceAccount(t *testing.T) { } func TestDeleteRateLimitServiceAccount(t *testing.T) { + rl := &egcfgv1a1.RateLimit{ + Backend: egcfgv1a1.RateLimitDatabaseBackend{ + Type: egcfgv1a1.RedisBackendType, + Redis: &egcfgv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + } + testCases := []struct { name string }{ @@ -120,16 +130,14 @@ func TestDeleteRateLimitServiceAccount(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - kube := &Infra{ - Client: fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build(), - Namespace: "test", - } - rateLimitInfra := new(ir.RateLimitInfra) + kube := newTestInfra(t) - err := kube.createOrUpdateRateLimitService(context.Background(), rateLimitInfra) + rateLimitInfra := new(ir.RateLimitInfra) + r := ratelimit.NewResourceRender(kube.Namespace, rateLimitInfra, rl, kube.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment) + err := kube.createOrUpdateServiceAccount(context.Background(), r) require.NoError(t, err) - err = kube.deleteRateLimitServiceAccount(context.Background(), rateLimitInfra) + err = kube.deleteServiceAccount(context.Background(), r) require.NoError(t, err) }) } diff --git a/internal/infrastructure/kubernetes/service.go b/internal/infrastructure/kubernetes/service.go deleted file mode 100644 index 5a893d0b187..00000000000 --- a/internal/infrastructure/kubernetes/service.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - "reflect" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - - egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" -) - -func (i *Infra) createOrUpdateService(ctx context.Context, svc *corev1.Service) error { - current := &corev1.Service{} - key := types.NamespacedName{ - Namespace: svc.Namespace, - Name: svc.Name, - } - - if err := i.Client.Get(ctx, key, current); err != nil { - // Create if not found. - if kerrors.IsNotFound(err) { - if err := i.Client.Create(ctx, svc); err != nil { - return fmt.Errorf("failed to create service %s/%s: %w", - svc.Namespace, svc.Name, err) - } - } - } else { - // Update if current value is different. - if !reflect.DeepEqual(svc.Spec, current.Spec) { - svc.ResourceVersion = current.ResourceVersion - svc.UID = current.UID - if err := i.Client.Update(ctx, svc); err != nil { - return fmt.Errorf("failed to update service %s/%s: %w", - svc.Namespace, svc.Name, err) - } - } - } - - return nil -} - -func (i *Infra) deleteService(ctx context.Context, svc *corev1.Service) error { - if err := i.Client.Delete(ctx, svc); err != nil { - if kerrors.IsNotFound(err) { - return nil - } - return fmt.Errorf("failed to delete service %s/%s: %w", svc.Namespace, svc.Name, err) - } - - return nil -} - -func expectedServiceSpec(serviceType *egcfgv1a1.ServiceType) corev1.ServiceSpec { - serviceSpec := corev1.ServiceSpec{} - serviceSpec.Type = corev1.ServiceType(*serviceType) - serviceSpec.SessionAffinity = corev1.ServiceAffinityNone - if *serviceType == egcfgv1a1.ServiceTypeLoadBalancer { - // Preserve the client source IP and avoid a second hop for LoadBalancer. - serviceSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal - } - return serviceSpec -} diff --git a/internal/infrastructure/kubernetes/service_test.go b/internal/infrastructure/kubernetes/service_test.go deleted file mode 100644 index 62f509f0a31..00000000000 --- a/internal/infrastructure/kubernetes/service_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/util/intstr" - - egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" -) - -func checkServiceHasPort(t *testing.T, svc *corev1.Service, port int32) { - t.Helper() - - for _, p := range svc.Spec.Ports { - if p.Port == port { - return - } - } - t.Errorf("service is missing port %q", port) -} - -func checkServiceHasTargetPort(t *testing.T, svc *corev1.Service, port int32) { - t.Helper() - - intStrPort := intstr.IntOrString{IntVal: port} - for _, p := range svc.Spec.Ports { - if p.TargetPort == intStrPort { - return - } - } - t.Errorf("service is missing targetPort %d", port) -} - -func checkServiceHasPortName(t *testing.T, svc *corev1.Service, name string) { - t.Helper() - - for _, p := range svc.Spec.Ports { - if p.Name == name { - return - } - } - t.Errorf("service is missing port name %q", name) -} - -func checkServiceHasLabels(t *testing.T, svc *corev1.Service, expected map[string]string) { - t.Helper() - - if apiequality.Semantic.DeepEqual(svc.Labels, expected) { - return - } - - t.Errorf("service has unexpected %q labels", svc.Labels) -} - -func checkServiceHasAnnotations(t *testing.T, svc *corev1.Service, expected map[string]string) { - t.Helper() - - if apiequality.Semantic.DeepEqual(svc.Annotations, expected) { - return - } - - t.Errorf("service has unexpected %q annotations", svc.Annotations) -} - -func checkServiceSpec(t *testing.T, svc *corev1.Service, expected corev1.ServiceSpec) { - t.Helper() - - expected.Ports = svc.Spec.Ports - expected.Selector = svc.Spec.Selector - if apiequality.Semantic.DeepEqual(svc.Spec, expected) { - return - } - - t.Errorf("service has unexpected %q spec", &svc.Spec) -} - -func TestExpectedServiceSpec(t *testing.T) { - type args struct { - serviceType *egcfgv1a1.ServiceType - } - tests := []struct { - name string - args args - want corev1.ServiceSpec - }{ - { - name: "LoadBalancer", - args: args{serviceType: egcfgv1a1.GetKubernetesServiceType(egcfgv1a1.ServiceTypeLoadBalancer)}, - want: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - SessionAffinity: corev1.ServiceAffinityNone, - ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, - }, - }, - { - name: "ClusterIP", - args: args{serviceType: egcfgv1a1.GetKubernetesServiceType(egcfgv1a1.ServiceTypeClusterIP)}, - want: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - SessionAffinity: corev1.ServiceAffinityNone, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, expectedServiceSpec(tt.args.serviceType), "expectedServiceSpec(%v)", tt.args.serviceType) - }) - } -} diff --git a/internal/infrastructure/kubernetes/serviceaccount.go b/internal/infrastructure/kubernetes/serviceaccount.go deleted file mode 100644 index 9cb3dfff0b3..00000000000 --- a/internal/infrastructure/kubernetes/serviceaccount.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package kubernetes - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" -) - -func (i *Infra) createOrUpdateServiceAccount(ctx context.Context, sa *corev1.ServiceAccount) error { - current := &corev1.ServiceAccount{} - key := types.NamespacedName{ - Namespace: sa.Namespace, - Name: sa.Name, - } - - if err := i.Client.Get(ctx, key, current); err != nil { - if kerrors.IsNotFound(err) { - // Create if it does not exist. - if err := i.Client.Create(ctx, sa); err != nil { - return fmt.Errorf("failed to create serviceaccount %s/%s: %w", - sa.Namespace, sa.Name, err) - } - } - } else { - // Since the ServiceAccount does not have a specific Spec field to compare - // just perform an update for now. - sa.ResourceVersion = current.ResourceVersion - sa.UID = current.UID - if err := i.Client.Update(ctx, sa); err != nil { - return fmt.Errorf("failed to update serviceaccount %s/%s: %w", - sa.Namespace, sa.Name, err) - } - } - - return nil -} - -func (i *Infra) deleteServiceAccount(ctx context.Context, sa *corev1.ServiceAccount) error { - if err := i.Client.Delete(ctx, sa); err != nil { - if kerrors.IsNotFound(err) { - return nil - } - return fmt.Errorf("failed to delete serviceaccount %s/%s: %w", sa.Namespace, sa.Name, err) - } - - return nil -} diff --git a/internal/infrastructure/kubernetes/utils/utils.go b/internal/infrastructure/kubernetes/utils/utils.go new file mode 100644 index 00000000000..bb23bfe8c93 --- /dev/null +++ b/internal/infrastructure/kubernetes/utils/utils.go @@ -0,0 +1,83 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package utils + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + providerutils "github.com/envoyproxy/gateway/internal/provider/utils" +) + +const ( + SdsCAFilename = "xds-trusted-ca.json" + SdsCertFilename = "xds-certificate.json" + // XdsTLSCertFilename is the fully qualified path of the file containing Envoy's + // xDS server TLS certificate. + XdsTLSCertFilename = "/certs/tls.crt" + // XdsTLSKeyFilename is the fully qualified path of the file containing Envoy's + // xDS server TLS key. + XdsTLSKeyFilename = "/certs/tls.key" + // XdsTLSCaFilename is the fully qualified path of the file containing Envoy's + // trusted CA certificate. + XdsTLSCaFilename = "/certs/ca.crt" +) + +var ( + // xDS certificate rotation is supported by using SDS path-based resource files. + SdsCAConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ + `"name":"xds_trusted_ca","validation_context":{"trusted_ca":{"filename":"%s"},`+ + `"match_typed_subject_alt_names":[{"san_type":"DNS","matcher":{"exact":"envoy-gateway"}}]}}]}`, XdsTLSCaFilename) + SdsCertConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ + `"name":"xds_certificate","tls_certificate":{"certificate_chain":{"filename":"%s"},`+ + `"private_key":{"filename":"%s"}}}]}`, XdsTLSCertFilename, XdsTLSKeyFilename) +) + +// GetSelector returns a label selector used to select resources +// based on the provided labels. +func GetSelector(labels map[string]string) *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: labels, + } +} + +// expectedResourceHashedName returns hashed resource name. +func ExpectedResourceHashedName(name string) string { + hashedName := providerutils.GetHashedName(name) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, hashedName) +} + +func ExpectedServiceSpec(serviceType *egcfgv1a1.ServiceType) corev1.ServiceSpec { + serviceSpec := corev1.ServiceSpec{} + serviceSpec.Type = corev1.ServiceType(*serviceType) + serviceSpec.SessionAffinity = corev1.ServiceAffinityNone + if *serviceType == egcfgv1a1.ServiceTypeLoadBalancer { + // Preserve the client source IP and avoid a second hop for LoadBalancer. + serviceSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + } + return serviceSpec +} + +// EnvoyAppLabel returns the labels used for all Envoy resources. +func EnvoyAppLabel() map[string]string { + return map[string]string{ + "app.gateway.envoyproxy.io/name": "envoy", + } +} + +// EnvoyLabels returns the labels, including extraLbls, used for Envoy resources. +func EnvoyLabels(extraLbls map[string]string) map[string]string { + lbls := EnvoyAppLabel() + for k, v := range extraLbls { + lbls[k] = v + } + + return lbls +} diff --git a/internal/infrastructure/kubernetes/utils/utils_test.go b/internal/infrastructure/kubernetes/utils/utils_test.go new file mode 100644 index 00000000000..7d6c9fbb4a8 --- /dev/null +++ b/internal/infrastructure/kubernetes/utils/utils_test.go @@ -0,0 +1,75 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" +) + +func TestExpectedServiceSpec(t *testing.T) { + type args struct { + serviceType *egcfgv1a1.ServiceType + } + tests := []struct { + name string + args args + want corev1.ServiceSpec + }{ + { + name: "LoadBalancer", + args: args{serviceType: egcfgv1a1.GetKubernetesServiceType(egcfgv1a1.ServiceTypeLoadBalancer)}, + want: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + SessionAffinity: corev1.ServiceAffinityNone, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + }, + }, + { + name: "ClusterIP", + args: args{serviceType: egcfgv1a1.GetKubernetesServiceType(egcfgv1a1.ServiceTypeClusterIP)}, + want: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + SessionAffinity: corev1.ServiceAffinityNone, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ExpectedServiceSpec(tt.args.serviceType), "expectedServiceSpec(%v)", tt.args.serviceType) + }) + } +} + +func TestEnvoyPodSelector(t *testing.T) { + cases := []struct { + name string + in map[string]string + expected map[string]string + }{ + { + name: "default", + in: map[string]string{"foo": "bar"}, + expected: map[string]string{ + "foo": "bar", + "app.gateway.envoyproxy.io/name": "envoy", + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run("", func(t *testing.T) { + got := GetSelector(EnvoyLabels(tc.in)) + require.Equal(t, tc.expected, got.MatchLabels) + }) + } +} diff --git a/internal/xds/translator/runner/runner.go b/internal/xds/translator/runner/runner.go index ef31fc200b6..c707e684e9f 100644 --- a/internal/xds/translator/runner/runner.go +++ b/internal/xds/translator/runner/runner.go @@ -10,7 +10,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" extension "github.com/envoyproxy/gateway/internal/extension/types" - infra "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/message" "github.com/envoyproxy/gateway/internal/xds/translator" @@ -65,7 +65,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { // Set the rate limit service URL if global rate limiting is enabled. if r.EnvoyGateway.RateLimit != nil { t.GlobalRateLimit = &translator.GlobalRateLimitSettings{ - ServiceURL: infra.GetRateLimitServiceURL(r.Namespace), + ServiceURL: ratelimit.GetServiceURL(r.Namespace), } } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index db98fc0311f..190f2a809d4 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -18,7 +18,7 @@ import ( "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/extension/testutils" - infra "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/xds/utils" ) @@ -138,7 +138,7 @@ func TestTranslateXds(t *testing.T) { ir := requireXdsIRFromInputTestData(t, "xds-ir", tc.name+".yaml") tr := &Translator{ GlobalRateLimit: &GlobalRateLimitSettings{ - ServiceURL: infra.GetRateLimitServiceURL("envoy-gateway-system"), + ServiceURL: ratelimit.GetServiceURL("envoy-gateway-system"), }, } @@ -244,7 +244,7 @@ func TestTranslateXdsWithExtension(t *testing.T) { ir := requireXdsIRFromInputTestData(t, "extension-xds-ir", tc.name+".yaml") tr := &Translator{ GlobalRateLimit: &GlobalRateLimitSettings{ - ServiceURL: infra.GetRateLimitServiceURL("envoy-gateway-system"), + ServiceURL: ratelimit.GetServiceURL("envoy-gateway-system"), }, } ext := v1alpha1.Extension{ From 733bc4124fa41c198cc97a69213d5ad22c773a19 Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Wed, 19 Apr 2023 12:20:33 +0800 Subject: [PATCH 2/3] refactor utils.go Signed-off-by: hejianpeng --- .../kubernetes/proxy/resource_provider.go | 34 +++++------ .../infrastructure/kubernetes/proxy/utils.go | 59 +++++++++++++++++++ .../kubernetes/proxy/utils_test.go | 32 ++++++++++ .../kubernetes/proxy_configmap_test.go | 9 ++- .../kubernetes/proxy_deployment_test.go | 3 +- .../kubernetes/proxy_infra_test.go | 12 ++-- .../kubernetes/proxy_serviceaccount_test.go | 3 +- .../infrastructure/kubernetes/utils/utils.go | 51 ---------------- .../kubernetes/utils/utils_test.go | 9 ++- 9 files changed, 126 insertions(+), 86 deletions(-) create mode 100644 internal/infrastructure/kubernetes/proxy/utils.go create mode 100644 internal/infrastructure/kubernetes/proxy/utils_test.go diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 84118e85606..715d38713dd 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -45,13 +45,13 @@ func NewResourceRender(ns string, infra *ir.Infra) *ResourceRender { } func (r *ResourceRender) Name() string { - return utils.ExpectedResourceHashedName(r.infra.Proxy.Name) + return ExpectedResourceHashedName(r.infra.Proxy.Name) } // ServiceAccount returns the expected proxy serviceAccount. func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { // Set the labels based on the owning gateway name. - labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + labels := EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { return nil, fmt.Errorf("missing owning gateway labels") } @@ -63,7 +63,7 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Name: ExpectedResourceHashedName(r.infra.Proxy.Name), Labels: labels, }, }, nil @@ -90,7 +90,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { } // Set the labels based on the owning gatewayclass name. - labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + labels := EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { return nil, fmt.Errorf("missing owning gateway labels") } @@ -113,7 +113,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Name: ExpectedResourceHashedName(r.infra.Proxy.Name), Labels: labels, Annotations: annotations, }, @@ -126,7 +126,7 @@ func (r *ResourceRender) Service() (*corev1.Service, error) { // ConfigMap returns the expected ConfigMap based on the provided infra. func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { // Set the labels based on the owning gateway name. - labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + labels := EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { return nil, fmt.Errorf("missing owning gateway labels") } @@ -138,12 +138,12 @@ func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Name: ExpectedResourceHashedName(r.infra.Proxy.Name), Labels: labels, }, Data: map[string]string{ - utils.SdsCAFilename: utils.SdsCAConfigMapData, - utils.SdsCertFilename: utils.SdsCertConfigMapData, + SdsCAFilename: SdsCAConfigMapData, + SdsCertFilename: SdsCertConfigMapData, }, }, nil } @@ -164,7 +164,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } // Set the labels based on the owning gateway name. - labels := utils.EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) + labels := EnvoyLabels(r.infra.GetProxyInfra().GetProxyMetadata().Labels) if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 { return nil, fmt.Errorf("missing owning gateway labels") } @@ -184,7 +184,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Name: ExpectedResourceHashedName(r.infra.Proxy.Name), Labels: labels, }, Spec: appsv1.DeploymentSpec{ @@ -197,7 +197,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, Spec: corev1.PodSpec{ Containers: containers, - ServiceAccountName: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + ServiceAccountName: ExpectedResourceHashedName(r.infra.Proxy.Name), AutomountServiceAccountToken: pointer.Bool(false), TerminationGracePeriodSeconds: pointer.Int64(int64(300)), DNSPolicy: corev1.DNSClusterFirst, @@ -218,16 +218,16 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: utils.ExpectedResourceHashedName(r.infra.Proxy.Name), + Name: ExpectedResourceHashedName(r.infra.Proxy.Name), }, Items: []corev1.KeyToPath{ { - Key: utils.SdsCAFilename, - Path: utils.SdsCAFilename, + Key: SdsCAFilename, + Path: SdsCAFilename, }, { - Key: utils.SdsCertFilename, - Path: utils.SdsCertFilename, + Key: SdsCertFilename, + Path: SdsCertFilename, }, }, DefaultMode: pointer.Int32(int32(420)), diff --git a/internal/infrastructure/kubernetes/proxy/utils.go b/internal/infrastructure/kubernetes/proxy/utils.go new file mode 100644 index 00000000000..2ff967d56df --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/utils.go @@ -0,0 +1,59 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package proxy + +import ( + "fmt" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" + providerutils "github.com/envoyproxy/gateway/internal/provider/utils" +) + +const ( + SdsCAFilename = "xds-trusted-ca.json" + SdsCertFilename = "xds-certificate.json" + // XdsTLSCertFilename is the fully qualified path of the file containing Envoy's + // xDS server TLS certificate. + XdsTLSCertFilename = "/certs/tls.crt" + // XdsTLSKeyFilename is the fully qualified path of the file containing Envoy's + // xDS server TLS key. + XdsTLSKeyFilename = "/certs/tls.key" + // XdsTLSCaFilename is the fully qualified path of the file containing Envoy's + // trusted CA certificate. + XdsTLSCaFilename = "/certs/ca.crt" +) + +var ( + // xDS certificate rotation is supported by using SDS path-based resource files. + SdsCAConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ + `"name":"xds_trusted_ca","validation_context":{"trusted_ca":{"filename":"%s"},`+ + `"match_typed_subject_alt_names":[{"san_type":"DNS","matcher":{"exact":"envoy-gateway"}}]}}]}`, XdsTLSCaFilename) + SdsCertConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ + `"name":"xds_certificate","tls_certificate":{"certificate_chain":{"filename":"%s"},`+ + `"private_key":{"filename":"%s"}}}]}`, XdsTLSCertFilename, XdsTLSKeyFilename) +) + +func ExpectedResourceHashedName(name string) string { + hashedName := providerutils.GetHashedName(name) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, hashedName) +} + +// EnvoyAppLabel returns the labels used for all Envoy resources. +func EnvoyAppLabel() map[string]string { + return map[string]string{ + "app.gateway.envoyproxy.io/name": "envoy", + } +} + +// EnvoyLabels returns the labels, including extraLbls, used for Envoy resources. +func EnvoyLabels(extraLbls map[string]string) map[string]string { + lbls := EnvoyAppLabel() + for k, v := range extraLbls { + lbls[k] = v + } + + return lbls +} diff --git a/internal/infrastructure/kubernetes/proxy/utils_test.go b/internal/infrastructure/kubernetes/proxy/utils_test.go new file mode 100644 index 00000000000..1dd56af5f87 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/utils_test.go @@ -0,0 +1,32 @@ +package proxy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEnvoyPodSelector(t *testing.T) { + cases := []struct { + name string + in map[string]string + expected map[string]string + }{ + { + name: "default", + in: map[string]string{"foo": "bar"}, + expected: map[string]string{ + "foo": "bar", + "app.gateway.envoyproxy.io/name": "envoy", + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run("", func(t *testing.T) { + got := EnvoyLabels(tc.in) + require.Equal(t, tc.expected, got) + }) + } +} diff --git a/internal/infrastructure/kubernetes/proxy_configmap_test.go b/internal/infrastructure/kubernetes/proxy_configmap_test.go index 1a7e775978d..adc7044efae 100644 --- a/internal/infrastructure/kubernetes/proxy_configmap_test.go +++ b/internal/infrastructure/kubernetes/proxy_configmap_test.go @@ -21,7 +21,6 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) @@ -52,8 +51,8 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) { }, }, Data: map[string]string{ - utils.SdsCAFilename: utils.SdsCAConfigMapData, - utils.SdsCertFilename: utils.SdsCertConfigMapData, + proxy.SdsCAFilename: proxy.SdsCAConfigMapData, + proxy.SdsCertFilename: proxy.SdsCertConfigMapData, }, }, }, @@ -82,8 +81,8 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) { }, }, Data: map[string]string{ - utils.SdsCAFilename: utils.SdsCAConfigMapData, - utils.SdsCertFilename: utils.SdsCertConfigMapData, + proxy.SdsCAFilename: proxy.SdsCAConfigMapData, + proxy.SdsCertFilename: proxy.SdsCertConfigMapData, }, }, }, diff --git a/internal/infrastructure/kubernetes/proxy_deployment_test.go b/internal/infrastructure/kubernetes/proxy_deployment_test.go index 29d8d20d8a7..4c171186344 100644 --- a/internal/infrastructure/kubernetes/proxy_deployment_test.go +++ b/internal/infrastructure/kubernetes/proxy_deployment_test.go @@ -21,7 +21,6 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) @@ -120,7 +119,7 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) { actual := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) diff --git a/internal/infrastructure/kubernetes/proxy_infra_test.go b/internal/infrastructure/kubernetes/proxy_infra_test.go index 26158e27bde..fe22d0c059a 100644 --- a/internal/infrastructure/kubernetes/proxy_infra_test.go +++ b/internal/infrastructure/kubernetes/proxy_infra_test.go @@ -20,7 +20,7 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" "github.com/envoyproxy/gateway/internal/ir" ) @@ -51,7 +51,7 @@ func newTestInfraWithClient(t *testing.T, cli client.Client) *Infra { func TestCreateProxyInfra(t *testing.T) { // Infra with Gateway owner labels. infraWithLabels := ir.NewInfra() - infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels = utils.EnvoyAppLabel() + infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels = proxy.EnvoyAppLabel() infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default" infraWithLabels.GetProxyInfra().GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = "test-gw" @@ -100,7 +100,7 @@ func TestCreateProxyInfra(t *testing.T) { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(sa), sa)) @@ -108,7 +108,7 @@ func TestCreateProxyInfra(t *testing.T) { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(cm), cm)) @@ -116,7 +116,7 @@ func TestCreateProxyInfra(t *testing.T) { deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(deploy), deploy)) @@ -124,7 +124,7 @@ func TestCreateProxyInfra(t *testing.T) { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(svc), svc)) diff --git a/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go b/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go index e0e9c208d73..e406f564e39 100644 --- a/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go +++ b/internal/infrastructure/kubernetes/proxy_serviceaccount_test.go @@ -22,7 +22,6 @@ import ( "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/proxy" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/utils" "github.com/envoyproxy/gateway/internal/ir" ) @@ -179,7 +178,7 @@ func TestCreateOrUpdateProxyServiceAccount(t *testing.T) { actual := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: kube.Namespace, - Name: utils.ExpectedResourceHashedName(tc.in.Proxy.Name), + Name: proxy.ExpectedResourceHashedName(tc.in.Proxy.Name), }, } require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)) diff --git a/internal/infrastructure/kubernetes/utils/utils.go b/internal/infrastructure/kubernetes/utils/utils.go index bb23bfe8c93..e7203d3ca89 100644 --- a/internal/infrastructure/kubernetes/utils/utils.go +++ b/internal/infrastructure/kubernetes/utils/utils.go @@ -6,38 +6,10 @@ package utils import ( - "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway/config" - providerutils "github.com/envoyproxy/gateway/internal/provider/utils" -) - -const ( - SdsCAFilename = "xds-trusted-ca.json" - SdsCertFilename = "xds-certificate.json" - // XdsTLSCertFilename is the fully qualified path of the file containing Envoy's - // xDS server TLS certificate. - XdsTLSCertFilename = "/certs/tls.crt" - // XdsTLSKeyFilename is the fully qualified path of the file containing Envoy's - // xDS server TLS key. - XdsTLSKeyFilename = "/certs/tls.key" - // XdsTLSCaFilename is the fully qualified path of the file containing Envoy's - // trusted CA certificate. - XdsTLSCaFilename = "/certs/ca.crt" -) - -var ( - // xDS certificate rotation is supported by using SDS path-based resource files. - SdsCAConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ - `"name":"xds_trusted_ca","validation_context":{"trusted_ca":{"filename":"%s"},`+ - `"match_typed_subject_alt_names":[{"san_type":"DNS","matcher":{"exact":"envoy-gateway"}}]}}]}`, XdsTLSCaFilename) - SdsCertConfigMapData = fmt.Sprintf(`{"resources":[{"@type":"type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",`+ - `"name":"xds_certificate","tls_certificate":{"certificate_chain":{"filename":"%s"},`+ - `"private_key":{"filename":"%s"}}}]}`, XdsTLSCertFilename, XdsTLSKeyFilename) ) // GetSelector returns a label selector used to select resources @@ -48,12 +20,6 @@ func GetSelector(labels map[string]string) *metav1.LabelSelector { } } -// expectedResourceHashedName returns hashed resource name. -func ExpectedResourceHashedName(name string) string { - hashedName := providerutils.GetHashedName(name) - return fmt.Sprintf("%s-%s", config.EnvoyPrefix, hashedName) -} - func ExpectedServiceSpec(serviceType *egcfgv1a1.ServiceType) corev1.ServiceSpec { serviceSpec := corev1.ServiceSpec{} serviceSpec.Type = corev1.ServiceType(*serviceType) @@ -64,20 +30,3 @@ func ExpectedServiceSpec(serviceType *egcfgv1a1.ServiceType) corev1.ServiceSpec } return serviceSpec } - -// EnvoyAppLabel returns the labels used for all Envoy resources. -func EnvoyAppLabel() map[string]string { - return map[string]string{ - "app.gateway.envoyproxy.io/name": "envoy", - } -} - -// EnvoyLabels returns the labels, including extraLbls, used for Envoy resources. -func EnvoyLabels(extraLbls map[string]string) map[string]string { - lbls := EnvoyAppLabel() - for k, v := range extraLbls { - lbls[k] = v - } - - return lbls -} diff --git a/internal/infrastructure/kubernetes/utils/utils_test.go b/internal/infrastructure/kubernetes/utils/utils_test.go index 7d6c9fbb4a8..1b5f88f71cb 100644 --- a/internal/infrastructure/kubernetes/utils/utils_test.go +++ b/internal/infrastructure/kubernetes/utils/utils_test.go @@ -49,7 +49,7 @@ func TestExpectedServiceSpec(t *testing.T) { } } -func TestEnvoyPodSelector(t *testing.T) { +func TestGetSelector(t *testing.T) { cases := []struct { name string in map[string]string @@ -57,7 +57,10 @@ func TestEnvoyPodSelector(t *testing.T) { }{ { name: "default", - in: map[string]string{"foo": "bar"}, + in: map[string]string{ + "foo": "bar", + "app.gateway.envoyproxy.io/name": "envoy", + }, expected: map[string]string{ "foo": "bar", "app.gateway.envoyproxy.io/name": "envoy", @@ -68,7 +71,7 @@ func TestEnvoyPodSelector(t *testing.T) { for _, tc := range cases { tc := tc t.Run("", func(t *testing.T) { - got := GetSelector(EnvoyLabels(tc.in)) + got := GetSelector(tc.in) require.Equal(t, tc.expected, got.MatchLabels) }) } From 20274eb613f468c80d1c27de139de3a7992803d4 Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Wed, 19 Apr 2023 12:24:30 +0800 Subject: [PATCH 3/3] fix license check Signed-off-by: hejianpeng --- internal/infrastructure/kubernetes/proxy/utils_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/infrastructure/kubernetes/proxy/utils_test.go b/internal/infrastructure/kubernetes/proxy/utils_test.go index 1dd56af5f87..e1def1a3821 100644 --- a/internal/infrastructure/kubernetes/proxy/utils_test.go +++ b/internal/infrastructure/kubernetes/proxy/utils_test.go @@ -1,3 +1,8 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + package proxy import (