From c864a40bc72e58213f35068ab5ef7bfed0f3b882 Mon Sep 17 00:00:00 2001 From: Shubham Chauhan Date: Sun, 6 Nov 2022 14:00:06 +0530 Subject: [PATCH] refactoring kubernetes provider to single reconciler Signed-off-by: Shubham Chauhan --- internal/gatewayapi/helpers_v1alpha2.go | 49 ++ internal/provider/kubernetes/controller.go | 488 +++++++++++++++++ internal/provider/kubernetes/gateway.go | 491 ++++-------------- internal/provider/kubernetes/gateway_test.go | 10 +- internal/provider/kubernetes/gatewayclass.go | 240 --------- .../provider/kubernetes/gatewayclass_test.go | 6 +- internal/provider/kubernetes/helpers.go | 189 +++++++ internal/provider/kubernetes/httproute.go | 376 -------------- .../provider/kubernetes/httproute_test.go | 379 +------------- internal/provider/kubernetes/kubernetes.go | 14 +- internal/provider/kubernetes/route.go | 421 +++++++++++++++ internal/provider/kubernetes/tlsroute.go | 353 ------------- 12 files changed, 1264 insertions(+), 1752 deletions(-) create mode 100644 internal/provider/kubernetes/controller.go delete mode 100644 internal/provider/kubernetes/gatewayclass.go delete mode 100644 internal/provider/kubernetes/httproute.go create mode 100644 internal/provider/kubernetes/route.go delete mode 100644 internal/provider/kubernetes/tlsroute.go diff --git a/internal/gatewayapi/helpers_v1alpha2.go b/internal/gatewayapi/helpers_v1alpha2.go index 1dc934a27f08..49213031e798 100644 --- a/internal/gatewayapi/helpers_v1alpha2.go +++ b/internal/gatewayapi/helpers_v1alpha2.go @@ -137,6 +137,55 @@ func DowngradeRouteParentStatuses(routeParentStatuses []v1beta1.RouteParentStatu return res } +// UpgradeBackendRef converts v1alpha2.BackendRef to v1beta1.BackendRef +func UpgradeBackendRef(old v1alpha2.BackendRef) v1beta1.BackendRef { + upgraded := v1beta1.BackendRef{} + + if old.Group != nil { + upgraded.Group = GroupPtr(string(*old.Group)) + } + + if old.Kind != nil { + upgraded.Kind = KindPtr(string(*old.Kind)) + } + + if old.Namespace != nil { + upgraded.Namespace = NamespacePtr(string(*old.Namespace)) + } + + upgraded.Name = v1beta1.ObjectName(old.Name) + + if old.Port != nil { + upgraded.Port = PortNumPtr(int32(*old.Port)) + } + + return upgraded +} + +func DowngradeBackendRef(old v1beta1.BackendRef) v1alpha2.BackendRef { + downgraded := v1alpha2.BackendRef{} + + if old.Group != nil { + downgraded.Group = GroupPtrV1Alpha2(string(*old.Group)) + } + + if old.Kind != nil { + downgraded.Kind = KindPtrV1Alpha2(string(*old.Kind)) + } + + if old.Namespace != nil { + downgraded.Namespace = NamespacePtrV1Alpha2(string(*old.Namespace)) + } + + downgraded.Name = v1alpha2.ObjectName(old.Name) + + if old.Port != nil { + downgraded.Port = PortNumPtrV1Alpha2(int(*old.Port)) + } + + return downgraded +} + func NamespaceDerefOrAlpha(namespace *v1alpha2.Namespace, defaultNamespace string) string { if namespace != nil && *namespace != "" { return string(*namespace) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go new file mode 100644 index 000000000000..80831b29ea6b --- /dev/null +++ b/internal/provider/kubernetes/controller.go @@ -0,0 +1,488 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package kubernetes + +import ( + "context" + "fmt" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/message" + "github.com/envoyproxy/gateway/internal/provider/utils" + "github.com/envoyproxy/gateway/internal/status" + "github.com/envoyproxy/gateway/internal/utils/slice" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +const ( + serviceTLSRouteIndex = "serviceTLSRouteBackendRef" + serviceHTTPRouteIndex = "serviceHTTPRouteBackendRef" + secretGatewayIndex = "secretGatewayIndex" +) + +type gatewayAPIReconciler struct { + client client.Client + log logr.Logger + statusUpdater status.Updater + classController gwapiv1b1.GatewayController + + resources *message.ProviderResources + referenceStore *providerReferenceStore +} + +// newGatewayAPIController +func newGatewayAPIController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources, referenceStore *providerReferenceStore) error { + ctx := context.Background() + + r := &gatewayAPIReconciler{ + client: mgr.GetClient(), + log: cfg.Logger, + classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), + statusUpdater: su, + resources: resources, + referenceStore: referenceStore, + } + + c, err := controller.New("gatewayapi", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + r.log.Info("created gatewayapi controller") + + // Subscribe to status updates + r.subscribeAndUpdateStatus(ctx) + + // Only enqueue GatewayClass objects that match this Envoy Gateway's controller name. + if err := c.Watch( + &source.Kind{Type: &gwapiv1b1.GatewayClass{}}, + &handler.EnqueueRequestForObject{}, + predicate.NewPredicateFuncs(r.hasMatchingController), + ); err != nil { + return err + } + + // Watch Gateway CRUDs and reconcile affected GatewayClass. + if err := c.Watch( + &source.Kind{Type: &gwapiv1b1.Gateway{}}, + handler.EnqueueRequestsFromMapFunc(r.processGateway), + ); err != nil { + return err + } + if err := addGatewayIndexers(ctx, mgr); err != nil { + return err + } + + // Watch HTTPRoute CRUDs and process affected Gateways. + if err := c.Watch( + &source.Kind{Type: &gwapiv1b1.HTTPRoute{}}, + handler.EnqueueRequestsFromMapFunc(r.processHTTPRoute), + ); err != nil { + return err + } + if err := addHTTPRouteIndexers(ctx, mgr); err != nil { + return err + } + + // Watch TLSRoute CRUDs and process affected Gateways. + if err := c.Watch( + &source.Kind{Type: &gwapiv1a2.TLSRoute{}}, + handler.EnqueueRequestsFromMapFunc(r.processTLSRoute), + ); err != nil { + return err + } + if err := addTLSRouteIndexers(ctx, mgr); err != nil { + return err + } + + // Watch Service CRUDs and process affected *Route objects. + if err := c.Watch( + &source.Kind{Type: &corev1.Service{}}, + handler.EnqueueRequestsFromMapFunc(r.processService), + ); err != nil { + return err + } + + // Watch Secret CRUDs and process affected Gateways. + if err := c.Watch( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.processSecret), + ); err != nil { + return err + } + + // Watch ReferenceGrant CRUDs and process affected Gateways. + if err := c.Watch( + &source.Kind{Type: &gwapiv1a2.ReferenceGrant{}}, + handler.EnqueueRequestsFromMapFunc(r.processReferenceGrant), + ); err != nil { + return err + } + + r.log.Info("watching gatewayAPI related objects") + return nil +} + +func (r *gatewayAPIReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + r.log.WithName(request.Name).Info("reconciling gatewayclass") + + var gatewayClasses gwapiv1b1.GatewayClassList + if err := r.client.List(ctx, &gatewayClasses); err != nil { + return reconcile.Result{}, fmt.Errorf("error listing gatewayclasses: %v", err) + } + + var cc controlledClasses + + for i := range gatewayClasses.Items { + if gatewayClasses.Items[i].Spec.ControllerName == r.classController { + // The gatewayclass was marked for deletion and the finalizer removed, + // so clean-up dependents. + if !gatewayClasses.Items[i].DeletionTimestamp.IsZero() && + !slice.ContainsString(gatewayClasses.Items[i].Finalizers, gatewayClassFinalizer) { + r.log.Info("gatewayclass marked for deletion") + cc.removeMatch(&gatewayClasses.Items[i]) + // Delete the gatewayclass from the watchable map. + r.resources.GatewayClasses.Delete(request.Name) + continue + } + + cc.addMatch(&gatewayClasses.Items[i]) + } + } + + // The gatewayclass was already deleted/finalized and there are stale queue entries. + acceptedGC := cc.acceptedClass() + if acceptedGC == nil { + r.log.Info("failed to find an accepted gatewayclass") + return reconcile.Result{}, nil + } + + // Store the accepted gatewayclass in the watchable map. + r.resources.GatewayClasses.Store(acceptedGC.GetName(), acceptedGC) + + updater := func(gc *gwapiv1b1.GatewayClass, accepted bool) error { + if r.statusUpdater != nil { + r.statusUpdater.Send(status.Update{ + NamespacedName: types.NamespacedName{Name: gc.Name}, + Resource: &gwapiv1b1.GatewayClass{}, + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + gc, ok := obj.(*gwapiv1b1.GatewayClass) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + + return status.SetGatewayClassAccepted(gc.DeepCopy(), accepted) + }), + }) + } else { + // this branch makes testing easier by not going through the status.Updater. + copy := status.SetGatewayClassAccepted(gc.DeepCopy(), accepted) + + if err := r.client.Status().Update(ctx, copy); err != nil { + return fmt.Errorf("error updating status of gatewayclass %s: %w", copy.Name, err) + } + } + return nil + } + + // Update status for all gateway classes + for _, gc := range cc.notAcceptedClasses() { + if err := updater(gc, false); err != nil { + return reconcile.Result{}, err + } + } + if acceptedGC != nil { + if err := updater(acceptedGC, true); err != nil { + return reconcile.Result{}, err + } + } + + r.log.WithName(request.Name).Info("reconciled gatewayclass") + return reconcile.Result{}, nil +} + +// hasMatchingController returns true if the provided object is a GatewayClass +// with a Spec.Controller string matching this Envoy Gateway's controller string, +// or false otherwise. +func (r *gatewayAPIReconciler) hasMatchingController(obj client.Object) bool { + gc, ok := obj.(*gwapiv1b1.GatewayClass) + if !ok { + r.log.Info("bypassing reconciliation due to unexpected object type", "type", obj) + return false + } + + if gc.Spec.ControllerName == r.classController { + r.log.Info("enqueueing gatewayclass") + return true + } + + r.log.Info("bypassing reconciliation due to controller name", "controller", gc.Spec.ControllerName) + return false +} + +// processSecret processes the Secret coming from the watcher and further +// processes parent Gateway objects to eventually reconcile GatewayClass. +func (r *gatewayAPIReconciler) processSecret(obj client.Object) []reconcile.Request { + r.log.Info("processing secret", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} + + secretkey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + secretDeleted := false + + secret := new(corev1.Secret) + if err := r.client.Get(ctx, secretkey, secret); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get secret") + return requests + } + + secretDeleted = true + // Remove the Secret from watchable map. + r.resources.Secrets.Delete(secretkey) + } + + if !secretDeleted { + // Store the Secret in watchable map. + r.resources.Secrets.Store(secretkey, secret.DeepCopy()) + } + + // Find the Gateways that reference this Secret. + gatewayList := &gwapiv1b1.GatewayList{} + if err := r.client.List(context.Background(), gatewayList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(secretGatewayIndex, utils.NamespacedName(obj).String()), + }); err != nil { + return requests + } + + for _, gw := range gatewayList.Items { + requests = append(requests, r.processGateway(gw.DeepCopy())...) + } + + return requests +} + +// processReferenceGrant processes the ReferenceGrant coming from the watcher +// and further processes parent Gateway objects to eventually reconcile GatewayClass. +func (r *gatewayAPIReconciler) processReferenceGrant(obj client.Object) []reconcile.Request { + r.log.Info("processing reference grant", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} + + refgrantkey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + gatewayReferences := []types.NamespacedName{} + refGrantDeleted := false + + refgrant := new(gwapiv1a2.ReferenceGrant) + if err := r.client.Get(ctx, refgrantkey, refgrant); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get reference grant") + return requests + } + + refGrantDeleted = true + // Remove the Reference Grant from watchable map. + if resourceRefGrant, found := r.resources.ReferenceGrants.Load(refgrantkey); found { + r.resources.ReferenceGrants.Delete(refgrantkey) + gatewayReferences = append(gatewayReferences, findGatewayReferencesFromRefGrant(resourceRefGrant)...) + } + } + + if !refGrantDeleted { + // Store the Reference Grant in watchable map. + r.resources.ReferenceGrants.Store(refgrantkey, refgrant.DeepCopy()) + gatewayReferences = append(gatewayReferences, findGatewayReferencesFromRefGrant(refgrant)...) + } + + for _, gwref := range gatewayReferences { + var gateway gwapiv1b1.Gateway + if err := r.client.Get(ctx, gwref, &gateway); err != nil { + return requests + } + requests = append(requests, r.processGateway(gateway.DeepCopy())...) + } + + return requests +} + +// addGatewayIndexers adds indexing on Gateway, for Secret objects that are +// referenced in Gateway objects. This helps in querying for Gateways that are +// affected by a particular Secret CRUD. +func addGatewayIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1b1.Gateway{}, secretGatewayIndex, func(rawObj client.Object) []string { + gateway := rawObj.(*gwapiv1b1.Gateway) + var secretReferences []string + for _, listener := range gateway.Spec.Listeners { + if listener.TLS == nil || *listener.TLS.Mode != gwapiv1b1.TLSModeTerminate { + continue + } + for _, cert := range listener.TLS.CertificateRefs { + if *cert.Kind == kindSecret { + // If an explicit Secret namespace is not provided, use the Gateway namespace to + // lookup the provided Secret Name. + secretReferences = append(secretReferences, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(cert.Namespace, gateway.Namespace), + Name: string(cert.Name), + }.String(), + ) + } + } + } + return secretReferences + }); err != nil { + return err + } + return nil +} + +// removeFinalizer removes the gatewayclass finalizer from the provided gc, if it exists. +func (r *gatewayAPIReconciler) removeFinalizer(ctx context.Context, gc *gwapiv1b1.GatewayClass) error { + if slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) { + updated := gc.DeepCopy() + updated.Finalizers = slice.RemoveString(updated.Finalizers, gatewayClassFinalizer) + if err := r.client.Update(ctx, updated); err != nil { + return fmt.Errorf("failed to remove finalizer from gatewayclass %s: %w", gc.Name, err) + } + } + return nil +} + +// addFinalizer adds the gatewayclass finalizer to the provided gc, if it doesn't exist. +func (r *gatewayAPIReconciler) addFinalizer(ctx context.Context, gc *gwapiv1b1.GatewayClass) error { + if !slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) { + updated := gc.DeepCopy() + updated.Finalizers = append(updated.Finalizers, gatewayClassFinalizer) + if err := r.client.Update(ctx, updated); err != nil { + return fmt.Errorf("failed to add finalizer to gatewayclass %s: %w", gc.Name, err) + } + } + return nil +} + +// acceptedClass returns the GatewayClass from the provided list that matches +// the configured controller name and contains the Accepted=true status condition. +func (r *gatewayAPIReconciler) acceptedClass(gcList *gwapiv1b1.GatewayClassList) *gwapiv1b1.GatewayClass { + if gcList == nil { + return nil + } + for i := range gcList.Items { + gc := &gcList.Items[i] + if gc.Spec.ControllerName == r.classController && isAccepted(gc) { + return gc + } + } + return nil +} + +// subscribeAndUpdateStatus subscribes to gateway API object status updates and +// writes it into the Kubernetes API Server. +func (r *gatewayAPIReconciler) subscribeAndUpdateStatus(ctx context.Context) { + // Gateway object status updater + go func() { + message.HandleSubscription(r.resources.GatewayStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1b1.Gateway]) { + // skip delete updates. + if update.Delete { + return + } + key := update.Key + val := update.Value + r.statusUpdater.Send(status.Update{ + NamespacedName: key, + Resource: new(gwapiv1b1.Gateway), + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + g, ok := obj.(*gwapiv1b1.Gateway) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + gCopy := g.DeepCopy() + gCopy.Status.Listeners = val.Status.Listeners + return gCopy + }), + }) + }, + ) + }() + + // HTTPRoute object status updater + go func() { + message.HandleSubscription(r.resources.HTTPRouteStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1b1.HTTPRoute]) { + // skip delete updates. + if update.Delete { + return + } + key := update.Key + val := update.Value + r.statusUpdater.Send(status.Update{ + NamespacedName: key, + Resource: new(gwapiv1b1.HTTPRoute), + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + h, ok := obj.(*gwapiv1b1.HTTPRoute) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + hCopy := h.DeepCopy() + hCopy.Status.Parents = val.Status.Parents + return hCopy + }), + }) + }, + ) + }() + + // TLSRoute object status updater + go func() { + message.HandleSubscription(r.resources.TLSRouteStatuses.Subscribe(ctx), + func(update message.Update[types.NamespacedName, *gwapiv1a2.TLSRoute]) { + // skip delete updates. + if update.Delete { + return + } + key := update.Key + val := update.Value + r.statusUpdater.Send(status.Update{ + NamespacedName: key, + Resource: new(gwapiv1a2.TLSRoute), + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + t, ok := obj.(*gwapiv1a2.TLSRoute) + if !ok { + panic(fmt.Sprintf("unsupported object type %T", obj)) + } + tCopy := t.DeepCopy() + tCopy.Status.Parents = val.Status.Parents + return tCopy + }), + }) + }, + ) + }() + + r.log.Info("status subscriber shutting down") +} diff --git a/internal/provider/kubernetes/gateway.go b/internal/provider/kubernetes/gateway.go index f7f958da0a81..2c4b5df5e2f6 100644 --- a/internal/provider/kubernetes/gateway.go +++ b/internal/provider/kubernetes/gateway.go @@ -16,269 +16,76 @@ import ( "context" "fmt" - "github.com/go-logr/logr" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/provider/utils" + "github.com/envoyproxy/gateway/internal/status" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/message" - "github.com/envoyproxy/gateway/internal/provider/utils" - "github.com/envoyproxy/gateway/internal/status" - "github.com/envoyproxy/gateway/internal/utils/slice" ) -const gatewayClassFinalizer = gwapiv1b1.GatewayClassFinalizerGatewaysExist - -type gatewayReconciler struct { - client client.Client - // classController is the configured gatewayclass controller name. - classController gwapiv1b1.GatewayController - statusUpdater status.Updater - log logr.Logger - - resources *message.ProviderResources -} - -// newGatewayController creates a gateway controller. The controller will watch for -// Gateway objects across all namespaces and reconcile those that match the configured -// gatewayclass controller name. -func newGatewayController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources) error { - r := &gatewayReconciler{ - client: mgr.GetClient(), - classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), - statusUpdater: su, - log: cfg.Logger, - resources: resources, - } - - c, err := controller.New("gateway", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - r.log.Info("created gateway controller") - - // Subscribe to status updates - go r.subscribeAndUpdateStatus(context.Background()) - - // Only enqueue Gateway objects that match this Envoy Gateway's controller name. - if err := c.Watch( - &source.Kind{Type: &gwapiv1b1.Gateway{}}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(r.hasMatchingController), - ); err != nil { - return err - } - r.log.Info("watching gateway objects") - - // Trigger gateway reconciliation when the Envoy Service or Deployment has changed. - if err := c.Watch(&source.Kind{Type: &corev1.Service{}}, r.enqueueRequestForOwningGateway()); err != nil { - return err - } - if err := c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, r.enqueueRequestForOwningGateway()); err != nil { - return err - } - // Trigger gateway reconciliation when a Secret that is referenced - // by a managed Gateway has changed. - if err := c.Watch(&source.Kind{Type: &corev1.Secret{}}, r.enqueueRequestForGatewaySecrets()); err != nil { - return err - } - // Trigger gateway reconciliation when a ReferenceGrant that refers - // to a managed Gateway has changed. - if err := c.Watch(&source.Kind{Type: &gwapiv1a2.ReferenceGrant{}}, r.enqueueRequestForReferencedGateway()); err != nil { - return err - } - - return nil -} - -// hasMatchingController returns true if the provided object is a Gateway -// using a GatewayClass matching the configured gatewayclass controller name. -func (r *gatewayReconciler) hasMatchingController(obj client.Object) bool { - gw, ok := obj.(*gwapiv1b1.Gateway) - if !ok { - r.log.Info("unexpected object type, bypassing reconciliation", "object", obj) - return false - } +// processGateway processes the Gateway coming from the watcher and eventually +// reconciles the parent GatewayClass. +func (r *gatewayAPIReconciler) processGateway(obj client.Object) []reconcile.Request { + r.log.Info("processing gateway", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} - gc := &gwapiv1b1.GatewayClass{} - key := types.NamespacedName{Name: string(gw.Spec.GatewayClassName)} - if err := r.client.Get(context.Background(), key, gc); err != nil { - r.log.Error(err, "failed to get gatewayclass", "name", gw.Spec.GatewayClassName) - return false - } - - if gc.Spec.ControllerName != r.classController { - r.log.Info("gatewayclass name for gateway doesn't match configured name", - "namespace", gw.Namespace, "name", gw.Name) - return false + gwkey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), } - return true -} - -// enqueueRequestForOwningGateway returns an event handler that maps events for -// resources with Gateway owning labels to reconcile requests for those Gateway objects. -func (r *gatewayReconciler) enqueueRequestForOwningGateway() handler.EventHandler { - return handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { - labels := a.GetLabels() - if labels == nil { - return nil - } - - gatewayNamespace := labels[gatewayapi.OwningGatewayNamespaceLabel] - gatewayName := labels[gatewayapi.OwningGatewayNameLabel] - - if len(gatewayNamespace) == 0 || len(gatewayName) == 0 { - return nil - } - - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: gatewayNamespace, - Name: gatewayName, - }, - }, - } - }) -} - -// enqueueRequestForGatewaySecrets returns an event handler that maps events for -// Secrets referenced by managed Gateways to reconcile requests for those Gateway objects. -func (r *gatewayReconciler) enqueueRequestForGatewaySecrets() handler.EventHandler { - return handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { - secret, ok := a.(*corev1.Secret) - if !ok { - r.log.Info("bypassing reconciliation due to unexpected object type", "type", a) - return nil - } - - ctx := context.Background() - var gateways gwapiv1b1.GatewayList - if err := r.client.List(ctx, &gateways); err != nil { - return nil - } - - var reqs []reconcile.Request - for i := range gateways.Items { - gw := gateways.Items[i] - if r.hasMatchingController(&gw) { - for j := range gw.Spec.Listeners { - if terminatesTLS(&gw.Spec.Listeners[j]) { - secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, &gw) - if err != nil { - return nil - } - for _, s := range secrets { - if s.Namespace == secret.Namespace && s.Name == secret.Name { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: gw.Namespace, - Name: gw.Name, - }, - } - reqs = append(reqs, req) - } - } - } - } - } - } - - return reqs - }) -} + gatewayDeleted := false + gatewayClasses := []string{} -// enqueueRequestForReferencedGateway returns an event handler that maps events for -// resources that reference a managed Gateway to reconcile requests for those Gateway objects. -// Note: A ReferenceGrant is the only supported object type. -func (r *gatewayReconciler) enqueueRequestForReferencedGateway() handler.EventHandler { - return handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { - rg, ok := a.(*gwapiv1a2.ReferenceGrant) - if !ok { - r.log.Info("bypassing reconciliation due to unexpected object type", "type", a) - return nil - } - - var refs []types.NamespacedName - for _, to := range rg.Spec.To { - if to.Group == gwapiv1a2.GroupName && - to.Kind == gatewayapi.KindGateway && - to.Name != nil { - ref := types.NamespacedName{Namespace: rg.Namespace, Name: string(*to.Name)} - refs = append(refs, ref) - } - } - for _, from := range rg.Spec.From { - if from.Group == gwapiv1a2.GroupName && - from.Kind == gatewayapi.KindGateway { - ref := types.NamespacedName{Namespace: string(from.Namespace), Name: rg.Name} - refs = append(refs, ref) - } + gw := new(gwapiv1b1.Gateway) + if err := r.client.Get(ctx, gwkey, gw); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get gateway") + return requests } - ctx := context.Background() - var gateways gwapiv1b1.GatewayList - if err := r.client.List(ctx, &gateways); err != nil { - return nil + gatewayDeleted = true + if resourceGateway, found := r.resources.Gateways.Load(gwkey); found { + gatewayClasses = append(gatewayClasses, string(resourceGateway.Spec.GatewayClassName)) + // Remove the Gateway from watchable map. + r.resources.Gateways.Delete(gwkey) } - - var reqs []reconcile.Request - for i := range gateways.Items { - gw := gateways.Items[i] - for _, ref := range refs { - if gw.Namespace == ref.Namespace && gw.Name == ref.Name && r.hasMatchingController(&gw) { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: gw.Namespace, - Name: gw.Name, - }, - } - reqs = append(reqs, req) - } - } - } - - return reqs - }) -} - -// Reconcile finds all the Gateways for the GatewayClass with an "Accepted: true" condition -// and passes all Gateways for the configured GatewayClass to the IR for processing. -func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - r.log.Info("reconciling gateway", "namespace", request.Namespace, "name", request.Name) + } allClasses := &gwapiv1b1.GatewayClassList{} if err := r.client.List(ctx, allClasses); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing gatewayclasses") + r.log.Error(err, "error listing gatewayclasses") + return requests } + // Find the GatewayClass for this controller with Accepted=true status condition. acceptedClass := r.acceptedClass(allClasses) if acceptedClass == nil { - r.log.Info("No accepted gatewayclass found for gateway", "namespace", request.Namespace, - "name", request.Name) - for namespacedName := range r.resources.Gateways.LoadAll() { - r.resources.Gateways.Delete(namespacedName) + r.log.Info("No accepted gatewayclass found for gateway", "namespace", gwkey.Namespace, + "name", gwkey.Name) + r.log.Info(fmt.Sprintf("heyo %v", r.resources == nil)) + if r.resources != nil { + for namespacedName := range r.resources.Gateways.LoadAll() { + r.resources.Gateways.Delete(namespacedName) + } } - return reconcile.Result{}, nil + return requests } allGateways := &gwapiv1b1.GatewayList{} if err := r.client.List(ctx, allGateways); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing gateways") + r.log.Error(err, "error listing gateways") + return requests } // Get all the Gateways for the Accepted=true GatewayClass. @@ -287,14 +94,16 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req r.log.Info("No gateways found for accepted gatewayclass") // If needed, remove the finalizer from the accepted GatewayClass. if err := r.removeFinalizer(ctx, acceptedClass); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from gatewayclass %s: %w", - acceptedClass.Name, err) + r.log.Error(err, fmt.Sprintf("failed to remove finalizer from gatewayclass %s", + acceptedClass.Name)) + return requests } } else { // If needed, finalize the accepted GatewayClass. if err := r.addFinalizer(ctx, acceptedClass); err != nil { - return reconcile.Result{}, fmt.Errorf("failed adding finalizer to gatewayclass %s: %w", - acceptedClass.Name, err) + r.log.Error(err, fmt.Sprintf("failed adding finalizer to gatewayclass %s", + acceptedClass.Name)) + return requests } } @@ -343,7 +152,7 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req } if err := r.client.Get(ctx, key, &refNs); err != nil { r.log.Info("failed to get referencegrant namespace", "name", refNs.Name) - return reconcile.Result{}, nil + return requests } r.resources.Namespaces.Store(refNs.Name, &refNs) } @@ -378,19 +187,14 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req if v, ok := r.resources.Gateways.Load(key); !ok || (gw.Generation > v.Generation) { r.resources.Gateways.Store(key, &gw) } - if key == request.NamespacedName { + if key == utils.NamespacedName(obj) { found = true } } if !found { - gw, ok := r.resources.Gateways.Load(request.NamespacedName) - if !ok { - r.log.Info("failed to find accepted gateway in the watchable map", "namespace", request.Namespace, "name", request.Name) - return reconcile.Result{}, nil - } + r.resources.Gateways.Delete(utils.NamespacedName(obj)) - r.resources.Gateways.Delete(request.NamespacedName) // Delete the TLS secrets from the resource map if no other managed // Gateways reference them. secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, gw) @@ -416,74 +220,32 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req } } - r.log.WithName(request.Namespace).WithName(request.Name).Info("reconciled gateway") - - return reconcile.Result{}, nil -} - -// acceptedClass returns the GatewayClass from the provided list that matches -// the configured controller name and contains the Accepted=true status condition. -func (r *gatewayReconciler) acceptedClass(gcList *gwapiv1b1.GatewayClassList) *gwapiv1b1.GatewayClass { - if gcList == nil { - return nil - } - for i := range gcList.Items { - gc := &gcList.Items[i] - if gc.Spec.ControllerName == r.classController && isAccepted(gc) { - return gc + if !gatewayDeleted { + // only store the resource if it does not exist or it has a newer spec. + if v, ok := r.resources.Gateways.Load(gwkey); !ok || (gw.Generation > v.Generation) { + r.resources.Gateways.Store(gwkey, gw.DeepCopy()) + r.log.Info("added gateway to watchable map") } - } - return nil -} -// isAccepted returns true if the provided gatewayclass contains the Accepted=true -// status condition. -func isAccepted(gc *gwapiv1b1.GatewayClass) bool { - if gc == nil { - return false - } - for _, cond := range gc.Status.Conditions { - if cond.Type == string(gwapiv1b1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { - return true + if len(gatewayClasses) == 0 || gatewayClasses[0] != string(gw.Spec.GatewayClassName) { + gatewayClasses = append(gatewayClasses, string(gw.Spec.GatewayClassName)) } } - return false -} -// gatewaysOfClass returns a list of gateways that reference gc from the provided gwList. -func gatewaysOfClass(gc *gwapiv1b1.GatewayClass, gwList *gwapiv1b1.GatewayList) []gwapiv1b1.Gateway { - var ret []gwapiv1b1.Gateway - if gwList == nil || gc == nil { - return ret - } - for i := range gwList.Items { - gw := gwList.Items[i] - if string(gw.Spec.GatewayClassName) == gc.Name { - ret = append(ret, gw) - } + // To handle the GatewayClassName update in the, both the old and new + // Gateway object are passed through this transformation function. + for _, gwclass := range gatewayClasses { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: gwclass}, + }) } - return ret -} -// envoyServiceForGateway returns the Envoy service, returning nil if the service doesn't exist. -func (r *gatewayReconciler) envoyServiceForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) (*corev1.Service, error) { - key := types.NamespacedName{ - Namespace: config.EnvoyGatewayNamespace, - Name: infraServiceName(gateway), - } - svc := new(corev1.Service) - if err := r.client.Get(ctx, key, svc); err != nil { - if kerrors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - return svc, nil + return requests } // gatewaysRefSecret returns true if a managed Gateway references the provided secret. // An error is returned if an error is encountered while checking. -func (r *gatewayReconciler) gatewaysRefSecret(ctx context.Context, secret *corev1.Secret) (bool, error) { +func (r *gatewayAPIReconciler) gatewaysRefSecret(ctx context.Context, secret *corev1.Secret) (bool, error) { if secret == nil { return false, fmt.Errorf("secret is nil") } @@ -493,7 +255,7 @@ func (r *gatewayReconciler) gatewaysRefSecret(ctx context.Context, secret *corev } for i := range gateways.Items { gw := gateways.Items[i] - if r.hasMatchingController(&gw) { + if r.hasMatchingControllerForGateway(&gw) { secrets, _, err := r.secretsAndRefGrantsForGateway(ctx, &gw) if err != nil { return false, err @@ -509,10 +271,35 @@ func (r *gatewayReconciler) gatewaysRefSecret(ctx context.Context, secret *corev return false, nil } +// hasMatchingControllerForGateway returns true if the provided object is a Gateway +// using a GatewayClass matching the configured gatewayclass controller name. +func (r *gatewayAPIReconciler) hasMatchingControllerForGateway(obj client.Object) bool { + gw, ok := obj.(*gwapiv1b1.Gateway) + if !ok { + r.log.Info("unexpected object type, bypassing reconciliation", "object", obj) + return false + } + + gc := &gwapiv1b1.GatewayClass{} + key := types.NamespacedName{Name: string(gw.Spec.GatewayClassName)} + if err := r.client.Get(context.Background(), key, gc); err != nil { + r.log.Error(err, "failed to get gatewayclass", "name", gw.Spec.GatewayClassName) + return false + } + + if gc.Spec.ControllerName != r.classController { + r.log.Info("gatewayclass name for gateway doesn't match configured name", + "namespace", gw.Namespace, "name", gw.Name) + return false + } + + return true +} + // secretsAndRefGrantsForGateway returns the Secrets referenced by the provided gateway listeners. // If the provided Gateway references a Secret in a different namespace, a list of // ReferenceGrants is returned that permit the cross namespace Secret reference. -func (r *gatewayReconciler) secretsAndRefGrantsForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) ([]corev1.Secret, []gwapiv1a2.ReferenceGrant, error) { +func (r *gatewayAPIReconciler) secretsAndRefGrantsForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) ([]corev1.Secret, []gwapiv1a2.ReferenceGrant, error) { var secrets []corev1.Secret var returnedGrants []gwapiv1a2.ReferenceGrant for i := range gateway.Spec.Listeners { @@ -581,50 +368,8 @@ func (r *gatewayReconciler) secretsAndRefGrantsForGateway(ctx context.Context, g return secrets, returnedGrants, nil } -// terminatesTLS returns true if the provided gateway contains a listener configured -// for TLS termination. -func terminatesTLS(listener *gwapiv1b1.Listener) bool { - if listener.TLS != nil && - listener.Protocol == gwapiv1b1.HTTPSProtocolType && - listener.TLS.Mode != nil && - *listener.TLS.Mode == gwapiv1b1.TLSModeTerminate { - return true - } - return false -} - -// refsSecret returns true if ref refers to a Secret. -func refsSecret(ref *gwapiv1b1.SecretObjectReference) bool { - return (ref.Group == nil || *ref.Group == corev1.GroupName) && - (ref.Kind == nil || *ref.Kind == gatewayapi.KindSecret) -} - -// addFinalizer adds the gatewayclass finalizer to the provided gc, if it doesn't exist. -func (r *gatewayReconciler) addFinalizer(ctx context.Context, gc *gwapiv1b1.GatewayClass) error { - if !slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) { - updated := gc.DeepCopy() - updated.Finalizers = append(updated.Finalizers, gatewayClassFinalizer) - if err := r.client.Update(ctx, updated); err != nil { - return fmt.Errorf("failed to add finalizer to gatewayclass %s: %w", gc.Name, err) - } - } - return nil -} - -// removeFinalizer removes the gatewayclass finalizer from the provided gc, if it exists. -func (r *gatewayReconciler) removeFinalizer(ctx context.Context, gc *gwapiv1b1.GatewayClass) error { - if slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) { - updated := gc.DeepCopy() - updated.Finalizers = slice.RemoveString(updated.Finalizers, gatewayClassFinalizer) - if err := r.client.Update(ctx, updated); err != nil { - return fmt.Errorf("failed to remove finalizer from gatewayclass %s: %w", gc.Name, err) - } - } - return nil -} - // envoyDeploymentForGateway returns the Envoy Deployment, returning nil if the Deployment doesn't exist. -func (r *gatewayReconciler) envoyDeploymentForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) (*appsv1.Deployment, error) { +func (r *gatewayAPIReconciler) envoyDeploymentForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) (*appsv1.Deployment, error) { key := types.NamespacedName{ Namespace: config.EnvoyGatewayNamespace, Name: infraDeploymentName(gateway), @@ -639,42 +384,18 @@ func (r *gatewayReconciler) envoyDeploymentForGateway(ctx context.Context, gatew return deployment, nil } -// subscribeAndUpdateStatus subscribes to gateway status updates and writes it into the -// Kubernetes API Server -func (r *gatewayReconciler) subscribeAndUpdateStatus(ctx context.Context) { - // Subscribe to resources - message.HandleSubscription(r.resources.GatewayStatuses.Subscribe(ctx), - func(update message.Update[types.NamespacedName, *gwapiv1b1.Gateway]) { - // skip delete updates. - if update.Delete { - return - } - key := update.Key - val := update.Value - r.statusUpdater.Send(status.Update{ - NamespacedName: key, - Resource: new(gwapiv1b1.Gateway), - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - g, ok := obj.(*gwapiv1b1.Gateway) - if !ok { - panic(fmt.Sprintf("unsupported object type %T", obj)) - } - gCopy := g.DeepCopy() - gCopy.Status.Listeners = val.Status.Listeners - return gCopy - }), - }) - }, - ) - r.log.Info("status subscriber shutting down") -} - -func infraServiceName(gateway *gwapiv1b1.Gateway) string { - infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) - return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) -} - -func infraDeploymentName(gateway *gwapiv1b1.Gateway) string { - infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) - return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) +// envoyServiceForGateway returns the Envoy service, returning nil if the service doesn't exist. +func (r *gatewayAPIReconciler) envoyServiceForGateway(ctx context.Context, gateway *gwapiv1b1.Gateway) (*corev1.Service, error) { + key := types.NamespacedName{ + Namespace: config.EnvoyGatewayNamespace, + Name: infraServiceName(gateway), + } + svc := new(corev1.Service) + if err := r.client.Get(ctx, key, svc); err != nil { + if kerrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return svc, nil } diff --git a/internal/provider/kubernetes/gateway_test.go b/internal/provider/kubernetes/gateway_test.go index 7d7e1fe2530f..0b7f2062b093 100644 --- a/internal/provider/kubernetes/gateway_test.go +++ b/internal/provider/kubernetes/gateway_test.go @@ -126,7 +126,7 @@ func TestGatewayHasMatchingController(t *testing.T) { // Create the reconciler. logger, err := log.NewLogger() require.NoError(t, err) - r := gatewayReconciler{ + r := gatewayAPIReconciler{ classController: v1alpha1.GatewayControllerName, log: logger, } @@ -135,7 +135,7 @@ func TestGatewayHasMatchingController(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { r.client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(match, nonMatch, tc.obj).Build() - actual := r.hasMatchingController(tc.obj) + actual := r.hasMatchingControllerForGateway(tc.obj) require.Equal(t, tc.expect, actual) }) } @@ -364,7 +364,7 @@ func TestAddFinalizer(t *testing.T) { } // Create the reconciler. - r := new(gatewayReconciler) + r := new(gatewayAPIReconciler) ctx := context.Background() for _, tc := range testCases { @@ -428,7 +428,7 @@ func TestRemoveFinalizer(t *testing.T) { } // Create the reconciler. - r := new(gatewayReconciler) + r := new(gatewayAPIReconciler) ctx := context.Background() for _, tc := range testCases { @@ -842,7 +842,7 @@ func TestSecretsAndRefGrantsForGateway(t *testing.T) { } // Create the reconciler. - r := new(gatewayReconciler) + r := new(gatewayAPIReconciler) ctx := context.Background() for i := range testCases { diff --git a/internal/provider/kubernetes/gatewayclass.go b/internal/provider/kubernetes/gatewayclass.go deleted file mode 100644 index 8a360202bb72..000000000000 --- a/internal/provider/kubernetes/gatewayclass.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -// TODO Portions of this code are based on code from Contour, available at: -// https://github.com/projectcontour/contour/blob/main/internal/controller/gatewayclass.go - -package kubernetes - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/message" - "github.com/envoyproxy/gateway/internal/status" - "github.com/envoyproxy/gateway/internal/utils/slice" -) - -type gatewayClassReconciler struct { - client client.Client - controller gwapiv1b1.GatewayController - statusUpdater status.Updater - log logr.Logger - - resources *message.ProviderResources -} - -// newGatewayClassController creates the gatewayclass controller. The controller -// will be pre-configured to watch for cluster-scoped GatewayClass objects with -// a controller field that matches name. -func newGatewayClassController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources) error { - r := &gatewayClassReconciler{ - client: mgr.GetClient(), - controller: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), - statusUpdater: su, - log: cfg.Logger, - resources: resources, - } - - c, err := controller.New("gatewayclass", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - r.log.Info("created gatewayclass controller") - - // Only enqueue GatewayClass objects that match this Envoy Gateway's controller name. - if err := c.Watch( - &source.Kind{Type: &gwapiv1b1.GatewayClass{}}, - &handler.EnqueueRequestForObject{}, - predicate.NewPredicateFuncs(r.hasMatchingController), - ); err != nil { - return err - } - r.log.Info("watching gatewayclass objects") - - return nil -} - -// hasMatchingController returns true if the provided object is a GatewayClass -// with a Spec.Controller string matching this Envoy Gateway's controller string, -// or false otherwise. -func (r *gatewayClassReconciler) hasMatchingController(obj client.Object) bool { - log := r.log.WithName(obj.GetName()) - - gc, ok := obj.(*gwapiv1b1.GatewayClass) - if !ok { - log.Info("bypassing reconciliation due to unexpected object type", "type", obj) - return false - } - - if gc.Spec.ControllerName == r.controller { - log.Info("enqueueing gatewayclass") - return true - } - - log.Info("bypassing reconciliation due to controller name", "controller", gc.Spec.ControllerName) - return false -} - -func (r *gatewayClassReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - r.log.WithName(request.Name).Info("reconciling gatewayclass") - - var gatewayClasses gwapiv1b1.GatewayClassList - if err := r.client.List(ctx, &gatewayClasses); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing gatewayclasses: %v", err) - } - - var cc controlledClasses - - for i := range gatewayClasses.Items { - if gatewayClasses.Items[i].Spec.ControllerName == r.controller { - // The gatewayclass was marked for deletion and the finalizer removed, - // so clean-up dependents. - if !gatewayClasses.Items[i].DeletionTimestamp.IsZero() && - !slice.ContainsString(gatewayClasses.Items[i].Finalizers, gatewayClassFinalizer) { - r.log.Info("gatewayclass marked for deletion") - cc.removeMatch(&gatewayClasses.Items[i]) - // Delete the gatewayclass from the watchable map. - r.resources.GatewayClasses.Delete(request.Name) - continue - } - - cc.addMatch(&gatewayClasses.Items[i]) - } - } - - // The gatewayclass was already deleted/finalized and there are stale queue entries. - acceptedGC := cc.acceptedClass() - if acceptedGC == nil { - r.log.Info("failed to find an accepted gatewayclass") - return reconcile.Result{}, nil - } - - // Store the accepted gatewayclass in the resource map. - r.resources.GatewayClasses.Store(acceptedGC.GetName(), acceptedGC) - - updater := func(gc *gwapiv1b1.GatewayClass, accepted bool) error { - if r.statusUpdater != nil { - r.statusUpdater.Send(status.Update{ - NamespacedName: types.NamespacedName{Name: gc.Name}, - Resource: &gwapiv1b1.GatewayClass{}, - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - gc, ok := obj.(*gwapiv1b1.GatewayClass) - if !ok { - panic(fmt.Sprintf("unsupported object type %T", obj)) - } - - return status.SetGatewayClassAccepted(gc.DeepCopy(), accepted) - }), - }) - } else { - // this branch makes testing easier by not going through the status.Updater. - copy := status.SetGatewayClassAccepted(gc.DeepCopy(), accepted) - - if err := r.client.Status().Update(ctx, copy); err != nil { - return fmt.Errorf("error updating status of gatewayclass %s: %w", copy.Name, err) - } - } - return nil - } - - // Update status for all gateway classes - for _, gc := range cc.notAcceptedClasses() { - if err := updater(gc, false); err != nil { - return reconcile.Result{}, err - } - } - if acceptedGC != nil { - if err := updater(acceptedGC, true); err != nil { - return reconcile.Result{}, err - } - } - - r.log.WithName(request.Name).Info("reconciled gatewayclass") - return reconcile.Result{}, nil -} - -type controlledClasses struct { - // matchedClasses holds all GatewayClass objects with matching controllerName. - matchedClasses []*gwapiv1b1.GatewayClass - - // oldestClass stores the first GatewayClass encountered with matching - // controllerName. This is maintained so that the oldestClass does not change - // during reboots. - oldestClass *gwapiv1b1.GatewayClass -} - -func (cc *controlledClasses) addMatch(gc *gwapiv1b1.GatewayClass) { - cc.matchedClasses = append(cc.matchedClasses, gc) - - switch { - case cc.oldestClass == nil: - cc.oldestClass = gc - case gc.CreationTimestamp.Time.Before(cc.oldestClass.CreationTimestamp.Time): - cc.oldestClass = gc - case gc.CreationTimestamp.Time.Equal(cc.oldestClass.CreationTimestamp.Time) && gc.Name < cc.oldestClass.Name: - // tie-breaker: first one in alphabetical order is considered oldest/accepted - cc.oldestClass = gc - } -} - -func (cc *controlledClasses) removeMatch(gc *gwapiv1b1.GatewayClass) { - // First remove gc from matchedClasses. - for i, matchedGC := range cc.matchedClasses { - if matchedGC.Name == gc.Name { - cc.matchedClasses[i] = cc.matchedClasses[len(cc.matchedClasses)-1] - cc.matchedClasses = cc.matchedClasses[:len(cc.matchedClasses)-1] - break - } - } - - // If the oldestClass is removed, find the new oldestClass candidate - // from matchedClasses. - if cc.oldestClass != nil && cc.oldestClass.Name == gc.Name { - if len(cc.matchedClasses) == 0 { - cc.oldestClass = nil - return - } - - cc.oldestClass = cc.matchedClasses[0] - for i := 1; i < len(cc.matchedClasses); i++ { - current := cc.matchedClasses[i] - if current.CreationTimestamp.Time.Before(cc.oldestClass.CreationTimestamp.Time) || - (current.CreationTimestamp.Time.Equal(cc.oldestClass.CreationTimestamp.Time) && - current.Name < cc.oldestClass.Name) { - cc.oldestClass = current - return - } - } - } -} - -func (cc *controlledClasses) acceptedClass() *gwapiv1b1.GatewayClass { - return cc.oldestClass -} - -func (cc *controlledClasses) notAcceptedClasses() []*gwapiv1b1.GatewayClass { - var res []*gwapiv1b1.GatewayClass - for _, gc := range cc.matchedClasses { - // skip the oldest one since it will be accepted. - if gc.Name != cc.oldestClass.Name { - res = append(res, gc) - } - } - - return res -} diff --git a/internal/provider/kubernetes/gatewayclass_test.go b/internal/provider/kubernetes/gatewayclass_test.go index 4200a2379e81..9f5b79a46c3f 100644 --- a/internal/provider/kubernetes/gatewayclass_test.go +++ b/internal/provider/kubernetes/gatewayclass_test.go @@ -53,9 +53,9 @@ func TestGatewayClassHasMatchingController(t *testing.T) { // Create the reconciler. logger, err := log.NewLogger() require.NoError(t, err) - r := gatewayClassReconciler{ - controller: v1alpha1.GatewayControllerName, - log: logger, + r := gatewayAPIReconciler{ + classController: v1alpha1.GatewayControllerName, + log: logger, } for _, tc := range testCases { diff --git a/internal/provider/kubernetes/helpers.go b/internal/provider/kubernetes/helpers.go index 7c94208b203f..ce2d89729084 100644 --- a/internal/provider/kubernetes/helpers.go +++ b/internal/provider/kubernetes/helpers.go @@ -9,10 +9,24 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/provider/utils" +) + +const ( + kindTLSRoute = "TLSRoute" + kindHTTPRoute = "HTTPRoute" + kindSecret = "Secret" + + gatewayClassFinalizer = gwapiv1b1.GatewayClassFinalizerGatewaysExist ) // validateParentRefs validates the provided routeParentReferences, returning the @@ -79,3 +93,178 @@ func isRoutePresentInNamespace(ctx context.Context, c client.Client, ns string) } return false, nil } + +type controlledClasses struct { + // matchedClasses holds all GatewayClass objects with matching controllerName. + matchedClasses []*gwapiv1b1.GatewayClass + + // oldestClass stores the first GatewayClass encountered with matching + // controllerName. This is maintained so that the oldestClass does not change + // during reboots. + oldestClass *gwapiv1b1.GatewayClass +} + +func (cc *controlledClasses) addMatch(gc *gwapiv1b1.GatewayClass) { + cc.matchedClasses = append(cc.matchedClasses, gc) + + switch { + case cc.oldestClass == nil: + cc.oldestClass = gc + case gc.CreationTimestamp.Time.Before(cc.oldestClass.CreationTimestamp.Time): + cc.oldestClass = gc + case gc.CreationTimestamp.Time.Equal(cc.oldestClass.CreationTimestamp.Time) && gc.Name < cc.oldestClass.Name: + // tie-breaker: first one in alphabetical order is considered oldest/accepted + cc.oldestClass = gc + } +} + +func (cc *controlledClasses) removeMatch(gc *gwapiv1b1.GatewayClass) { + // First remove gc from matchedClasses. + for i, matchedGC := range cc.matchedClasses { + if matchedGC.Name == gc.Name { + cc.matchedClasses[i] = cc.matchedClasses[len(cc.matchedClasses)-1] + cc.matchedClasses = cc.matchedClasses[:len(cc.matchedClasses)-1] + break + } + } + + // If the oldestClass is removed, find the new oldestClass candidate + // from matchedClasses. + if cc.oldestClass != nil && cc.oldestClass.Name == gc.Name { + if len(cc.matchedClasses) == 0 { + cc.oldestClass = nil + return + } + + cc.oldestClass = cc.matchedClasses[0] + for i := 1; i < len(cc.matchedClasses); i++ { + current := cc.matchedClasses[i] + if current.CreationTimestamp.Time.Before(cc.oldestClass.CreationTimestamp.Time) || + (current.CreationTimestamp.Time.Equal(cc.oldestClass.CreationTimestamp.Time) && + current.Name < cc.oldestClass.Name) { + cc.oldestClass = current + return + } + } + } +} + +func (cc *controlledClasses) acceptedClass() *gwapiv1b1.GatewayClass { + return cc.oldestClass +} + +func (cc *controlledClasses) notAcceptedClasses() []*gwapiv1b1.GatewayClass { + var res []*gwapiv1b1.GatewayClass + for _, gc := range cc.matchedClasses { + // skip the oldest one since it will be accepted. + if gc.Name != cc.oldestClass.Name { + res = append(res, gc) + } + } + + return res +} + +// isAccepted returns true if the provided gatewayclass contains the Accepted=true +// status condition. +func isAccepted(gc *gwapiv1b1.GatewayClass) bool { + if gc == nil { + return false + } + for _, cond := range gc.Status.Conditions { + if cond.Type == string(gwapiv1b1.GatewayClassConditionStatusAccepted) && cond.Status == metav1.ConditionTrue { + return true + } + } + return false +} + +// gatewaysOfClass returns a list of gateways that reference gc from the provided gwList. +func gatewaysOfClass(gc *gwapiv1b1.GatewayClass, gwList *gwapiv1b1.GatewayList) []gwapiv1b1.Gateway { + var ret []gwapiv1b1.Gateway + if gwList == nil || gc == nil { + return ret + } + for i := range gwList.Items { + gw := gwList.Items[i] + if string(gw.Spec.GatewayClassName) == gc.Name { + ret = append(ret, gw) + } + } + return ret +} + +// terminatesTLS returns true if the provided gateway contains a listener configured +// for TLS termination. +func terminatesTLS(listener *gwapiv1b1.Listener) bool { + if listener.TLS != nil && + listener.Protocol == gwapiv1b1.HTTPSProtocolType && + listener.TLS.Mode != nil && + *listener.TLS.Mode == gwapiv1b1.TLSModeTerminate { + return true + } + return false +} + +// refsSecret returns true if ref refers to a Secret. +func refsSecret(ref *gwapiv1b1.SecretObjectReference) bool { + return (ref.Group == nil || *ref.Group == corev1.GroupName) && + (ref.Kind == nil || *ref.Kind == gatewayapi.KindSecret) +} + +func infraServiceName(gateway *gwapiv1b1.Gateway) string { + infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) +} + +func infraDeploymentName(gateway *gwapiv1b1.Gateway) string { + infraName := utils.GetHashedName(fmt.Sprintf("%s-%s", gateway.Namespace, gateway.Name)) + return fmt.Sprintf("%s-%s", config.EnvoyPrefix, infraName) +} + +// findGatewayReferencesFromRefGrant helps in finding and aggregating all the +// Gateway references if present in the ReferenceGrant object ref. +func findGatewayReferencesFromRefGrant(ref *gwapiv1a2.ReferenceGrant) []types.NamespacedName { + refs := []types.NamespacedName{} + + for _, to := range ref.Spec.To { + if to.Group == gwapiv1a2.GroupName && to.Kind == gatewayapi.KindGateway && to.Name != nil { + refs = append(refs, types.NamespacedName{ + Namespace: ref.Namespace, + Name: string(*to.Name), + }) + } + } + for _, from := range ref.Spec.From { + if from.Group == gwapiv1a2.GroupName && from.Kind == gatewayapi.KindGateway { + refs = append(refs, types.NamespacedName{ + Namespace: string(from.Namespace), + Name: ref.Name, + }) + } + } + + return refs +} + +// validateBackendRef validates that ref is a reference to a local Service. +// TODO: Add support for: +// - Validating weights. +// - Validating ports. +// - Referencing HTTPRoutes. +// - Referencing Services/HTTPRoutes from other namespaces using ReferenceGrant. +func validateBackendRef(ref *gwapiv1b1.BackendRef) error { + switch { + case ref == nil: + return nil + case ref.Group != nil && *ref.Group != corev1.GroupName: + return fmt.Errorf("invalid group; must be nil or empty string") + case ref.Kind != nil && *ref.Kind != gatewayapi.KindService: + return fmt.Errorf("invalid kind %q; must be %q", + *ref.BackendObjectReference.Kind, gatewayapi.KindService) + case ref.Namespace != nil: + return fmt.Errorf("invalid namespace; must be nil") + } + + return nil +} diff --git a/internal/provider/kubernetes/httproute.go b/internal/provider/kubernetes/httproute.go deleted file mode 100644 index e99ef92cbf64..000000000000 --- a/internal/provider/kubernetes/httproute.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -// This file contains code derived from Contour, -// https://github.com/projectcontour/contour -// from the source file -// https://github.com/projectcontour/contour/blob/main/internal/controller/httproute.go -// and is provided here subject to the following: -// Copyright Project Contour Authors -// SPDX-License-Identifier: Apache-2.0 - -package kubernetes - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/message" - "github.com/envoyproxy/gateway/internal/provider/utils" - "github.com/envoyproxy/gateway/internal/status" -) - -const ( - kindHTTPRoute = "HTTPRoute" - - serviceHTTPRouteIndex = "serviceHTTPRouteBackendRef" -) - -type httpRouteReconciler struct { - client client.Client - log logr.Logger - statusUpdater status.Updater - classController gwapiv1b1.GatewayController - - resources *message.ProviderResources - referenceStore *providerReferenceStore -} - -// newHTTPRouteController creates the httproute controller from mgr. The controller will be pre-configured -// to watch for HTTPRoute objects across all namespaces. -func newHTTPRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources, referenceStore *providerReferenceStore) error { - r := &httpRouteReconciler{ - client: mgr.GetClient(), - log: cfg.Logger, - classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), - statusUpdater: su, - resources: resources, - referenceStore: referenceStore, - } - - c, err := controller.New("httproute", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - r.log.Info("created httproute controller") - - if err := c.Watch(&source.Kind{Type: &gwapiv1b1.HTTPRoute{}}, &handler.EnqueueRequestForObject{}); err != nil { - return err - } - - // Subscribe to status updates - go r.subscribeAndUpdateStatus(context.Background()) - - // Add indexing on HTTPRoute, for Service objects that are referenced in HTTPRoute objects - // via `.spec.rules.backendRefs`. This helps in querying for HTTPRoutes that are affected by - // a particular Service CRUD. - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &gwapiv1b1.HTTPRoute{}, serviceHTTPRouteIndex, func(rawObj client.Object) []string { - httpRoute := rawObj.(*gwapiv1b1.HTTPRoute) - var backendServices []string - for _, rule := range httpRoute.Spec.Rules { - for _, backend := range rule.BackendRefs { - if string(*backend.Kind) == gatewayapi.KindService { - // If an explicit Service namespace is not provided, use the HTTPRoute namespace to - // lookup the provided Service Name. - backendServices = append(backendServices, - types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, httpRoute.Namespace), - Name: string(backend.Name), - }.String(), - ) - } - } - } - return backendServices - }); err != nil { - return err - } - - // Watch Gateway CRUDs and reconcile affected HTTPRoutes. - if err := c.Watch( - &source.Kind{Type: &gwapiv1b1.Gateway{}}, - handler.EnqueueRequestsFromMapFunc(r.getHTTPRoutesForGateway), - ); err != nil { - return err - } - - // Watch Service CRUDs and reconcile affected HTTPRoutes. - if err := c.Watch( - &source.Kind{Type: &corev1.Service{}}, - handler.EnqueueRequestsFromMapFunc(r.getHTTPRoutesForService), - ); err != nil { - return err - } - - r.log.Info("watching httproute objects") - return nil -} - -// getHTTPRoutesForGateway uses a Gateway obj to fetch HTTPRoutes, iterating -// through them and creating a reconciliation request for each valid HTTPRoute -// that references obj. -func (r *httpRouteReconciler) getHTTPRoutesForGateway(obj client.Object) []reconcile.Request { - ctx := context.Background() - - gw, ok := obj.(*gwapiv1b1.Gateway) - if !ok { - r.log.Info("unexpected object type, bypassing reconciliation", "object", obj) - return []reconcile.Request{} - } - - routes := &gwapiv1b1.HTTPRouteList{} - if err := r.client.List(ctx, routes); err != nil { - return []reconcile.Request{} - } - - requests := []reconcile.Request{} - for i := range routes.Items { - route := routes.Items[i] - gateways, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, route.Spec.ParentRefs) - if err != nil { - r.log.Info("invalid parentRefs for httproute, bypassing reconciliation", "object", obj) - continue - } - for j := range gateways { - if gateways[j].Namespace == gw.Namespace && gateways[j].Name == gw.Name { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: route.Namespace, - Name: route.Name, - }, - } - requests = append(requests, req) - break - } - } - } - - return requests -} - -// getHTTPRoutesForService uses a Service obj to fetch HTTPRoutes that references -// the Service using `.spec.rules.backendRefs`. The affected HTTPRoutes are then -// pushed for reconciliation. -func (r *httpRouteReconciler) getHTTPRoutesForService(obj client.Object) []reconcile.Request { - affectedHTTPRouteList := &gwapiv1b1.HTTPRouteList{} - - if err := r.client.List(context.Background(), affectedHTTPRouteList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(serviceHTTPRouteIndex, utils.NamespacedName(obj).String()), - }); err != nil { - return []reconcile.Request{} - } - - requests := make([]reconcile.Request, len(affectedHTTPRouteList.Items)) - for i, item := range affectedHTTPRouteList.Items { - item := item - requests[i] = reconcile.Request{ - NamespacedName: utils.NamespacedName(&item), - } - } - - return requests -} - -func (r *httpRouteReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log := r.log.WithValues("namespace", request.Namespace, "name", request.Name) - - log.Info("reconciling httproute") - - // Fetch all HTTPRoutes from the cache. - routeList := &gwapiv1b1.HTTPRouteList{} - if err := r.client.List(ctx, routeList); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing httproutes") - } - - found := false - for i := range routeList.Items { - // See if this route from the list matched the reconciled route. - route := routeList.Items[i] - routeKey := utils.NamespacedName(&route) - if routeKey == request.NamespacedName { - found = true - } - - // Validate the route. - gws, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, route.Spec.ParentRefs) - if err != nil { - // Remove the route from the watchable map since it's invalid. - r.resources.HTTPRoutes.Delete(routeKey) - r.log.Error(err, "invalid parentRefs for httproute") - return reconcile.Result{}, nil - } - log.Info("validated httproute parentRefs") - - if len(gws) == 0 { - // Remove the route from the watchable map since it doesn't reference - // a managed Gateway. - log.Info("httproute doesn't reference any managed gateways") - r.resources.HTTPRoutes.Delete(routeKey) - return reconcile.Result{}, nil - } - - // only store the resource if it does not exist or it has a newer spec. - if v, ok := r.resources.HTTPRoutes.Load(routeKey); !ok || (route.Generation > v.Generation) { - r.resources.HTTPRoutes.Store(routeKey, &route) - log.Info("added httproute to resource map") - } - // Get the route's namespace from the cache. - nsKey := types.NamespacedName{Name: route.Namespace} - ns := new(corev1.Namespace) - if err := r.client.Get(ctx, nsKey, ns); err != nil { - if errors.IsNotFound(err) { - // The route's namespace doesn't exist in the cache, so remove it from - // the namespace resource map if it exists. - if _, ok := r.resources.Namespaces.Load(nsKey.Name); ok { - r.resources.Namespaces.Delete(nsKey.Name) - log.Info("deleted namespace from resource map") - } - } - return reconcile.Result{}, fmt.Errorf("failed to get namespace %s", nsKey.Name) - } - - // The route's namespace exists, so add it to the resource map. - r.resources.Namespaces.Store(nsKey.Name, ns) - log.Info("added namespace to resource map") - - // Get the route's backendRefs from the cache. Note that a Service is the - // only supported kind. - for i := range route.Spec.Rules { - for j := range route.Spec.Rules[i].BackendRefs { - ref := route.Spec.Rules[i].BackendRefs[j] - if err := validateBackendRef(&ref); err != nil { - return reconcile.Result{}, fmt.Errorf("invalid backendRef: %w", err) - } - - // The backendRef is valid, so get the referenced service from the cache. - svcKey := types.NamespacedName{Namespace: route.Namespace, Name: string(ref.Name)} - svc := new(corev1.Service) - if err := r.client.Get(ctx, svcKey, svc); err != nil { - if errors.IsNotFound(err) { - // The ref's service doesn't exist in the cache, so remove it from - // the resource map if it exists. - if _, ok := r.resources.Services.Load(svcKey); ok { - r.resources.Services.Delete(svcKey) - r.referenceStore.removeRouteToServicesMapping( - ObjectKindNamespacedName{kindHTTPRoute, route.Namespace, route.Name}, - svcKey, - ) - log.Info("deleted service from resource map") - } - } - return reconcile.Result{}, fmt.Errorf("failed to get service %s/%s", - svcKey.Namespace, svcKey.Name) - } - - // The backendRef Service exists, so add it to the resource map. - r.resources.Services.Store(svcKey, svc) - r.referenceStore.updateRouteToServicesMapping( - ObjectKindNamespacedName{kindHTTPRoute, route.Namespace, route.Name}, - svcKey, - ) - log.Info("added service to resource map") - } - } - } - - if !found { - // Delete the httproute from the resource map. - r.resources.HTTPRoutes.Delete(request.NamespacedName) - log.Info("deleted httproute from resource map") - - // Delete the Namespace and Service from the resource maps if no other - // routes (TLSRoute or HTTPRoute) exist in the namespace. - found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace) - if err != nil { - return reconcile.Result{}, err - } - if !found { - r.resources.Namespaces.Delete(request.Namespace) - log.Info("deleted namespace from resource map") - } - - // Delete the Service from the resource maps if no other - // routes (TLSRoute or HTTPRoute) reference that Service. - routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}) - for svc := range routeServices { - r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, request.Namespace, request.Name}, svc) - if !r.referenceStore.isServiceReferredByRoutes(svc) { - r.resources.Services.Delete(svc) - log.Info("deleted service from resource map", "namespace", svc.Namespace, "name", svc.Name) - } - } - } - - log.Info("reconciled httproute") - - return reconcile.Result{}, nil -} - -// validateBackendRef validates that ref is a reference to a local Service. -// TODO: Add support for: -// - Validating weights. -// - Validating ports. -// - Referencing HTTPRoutes. -// - Referencing Services/HTTPRoutes from other namespaces using ReferenceGrant. -func validateBackendRef(ref *gwapiv1b1.HTTPBackendRef) error { - switch { - case ref == nil: - return nil - case ref.Group != nil && *ref.Group != corev1.GroupName: - return fmt.Errorf("invalid group; must be nil or empty string") - case ref.Kind != nil && *ref.Kind != gatewayapi.KindService: - return fmt.Errorf("invalid kind %q; must be %q", - *ref.BackendRef.BackendObjectReference.Kind, gatewayapi.KindService) - case ref.Namespace != nil: - return fmt.Errorf("invalid namespace; must be nil") - } - - return nil -} - -// subscribeAndUpdateStatus subscribes to httproute status updates and writes it into the -// Kubernetes API Server -func (r *httpRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { - // Subscribe to resources - message.HandleSubscription(r.resources.HTTPRouteStatuses.Subscribe(ctx), - func(update message.Update[types.NamespacedName, *gwapiv1b1.HTTPRoute]) { - // skip delete updates. - if update.Delete { - return - } - key := update.Key - val := update.Value - r.statusUpdater.Send(status.Update{ - NamespacedName: key, - Resource: new(gwapiv1b1.HTTPRoute), - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - h, ok := obj.(*gwapiv1b1.HTTPRoute) - if !ok { - panic(fmt.Sprintf("unsupported object type %T", obj)) - } - hCopy := h.DeepCopy() - hCopy.Status.Parents = val.Status.Parents - return hCopy - }), - }) - }, - ) - r.log.Info("status subscriber shutting down") -} diff --git a/internal/provider/kubernetes/httproute_test.go b/internal/provider/kubernetes/httproute_test.go index 605213984d64..ac973a079a04 100644 --- a/internal/provider/kubernetes/httproute_test.go +++ b/internal/provider/kubernetes/httproute_test.go @@ -12,392 +12,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/envoyproxy/gateway/api/config/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/log" ) -func TestGetHTTPRoutesForGateway(t *testing.T) { - testCases := []struct { - name string - obj client.Object - routes []gwapiv1b1.HTTPRoute - classes []gwapiv1b1.GatewayClass - expect []reconcile.Request - }{ - { - name: "valid route", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - }, - }, - expect: []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: "test", - Name: "h1", - }, - }, - }, - }, - { - name: "one valid route in different namespace", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test2", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - Namespace: gatewayapi.NamespacePtr("test"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - }, - }, - expect: []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: "test2", - Name: "h1", - }, - }, - }, - }, - { - name: "two valid routes", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - }, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h2", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - }, - }, - expect: []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: "test", - Name: "h1", - }, - }, - { - NamespacedName: types.NamespacedName{ - Namespace: "test", - Name: "h2", - }, - }, - }, - }, - { - name: "object referenced unmanaged gateway", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController("unmanaged.controller"), - }, - }, - }, - expect: []reconcile.Request{}, - }, - { - name: "valid route", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("Gateway"), - Name: gwapiv1b1.ObjectName("gw1"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - }, - }, - expect: []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: "test", - Name: "h1", - }, - }, - }, - }, - { - name: "no valid routes", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - Spec: gwapiv1b1.GatewaySpec{ - GatewayClassName: "gc1", - }, - }, - routes: []gwapiv1b1.HTTPRoute{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h1", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("UnsupportedKind"), - Name: gwapiv1b1.ObjectName("unsupported"), - }, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "h2", - }, - Spec: gwapiv1b1.HTTPRouteSpec{ - CommonRouteSpec: gwapiv1b1.CommonRouteSpec{ - ParentRefs: []gwapiv1b1.ParentReference{ - { - Group: gatewayapi.GroupPtr(gwapiv1b1.GroupName), - Kind: gatewayapi.KindPtr("UnsupportedKind"), - Name: gwapiv1b1.ObjectName("unsupported2"), - }, - }, - }, - }, - }, - }, - classes: []gwapiv1b1.GatewayClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - Spec: gwapiv1b1.GatewayClassSpec{ - ControllerName: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName), - }, - }, - }, - expect: []reconcile.Request{}, - }, - { - name: "no routes", - obj: &gwapiv1b1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test", - Name: "gw1", - }, - }, - expect: []reconcile.Request{}, - }, - { - name: "invalid object type", - obj: &gwapiv1b1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gc1", - }, - }, - expect: []reconcile.Request{}, - }, - } - - // Create the reconciler. - logger, err := log.NewLogger() - require.NoError(t, err) - r := &httpRouteReconciler{ - log: logger, - classController: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName)} - - for i := range testCases { - tc := testCases[i] - t.Run(tc.name, func(t *testing.T) { - objs := []client.Object{tc.obj} - for i := range tc.routes { - objs = append(objs, &tc.routes[i]) - } - for i := range tc.classes { - objs = append(objs, &tc.classes[i]) - } - r.client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(objs...).Build() - reqs := r.getHTTPRoutesForGateway(tc.obj) - assert.Equal(t, tc.expect, reqs) - }) - } -} - func TestValidateParentRefs(t *testing.T) { testCases := []struct { name string @@ -723,7 +346,7 @@ func TestValidateParentRefs(t *testing.T) { } // Create the reconciler. - r := &httpRouteReconciler{classController: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName)} + r := &gatewayAPIReconciler{classController: gwapiv1b1.GatewayController(v1alpha1.GatewayControllerName)} ctx := context.Background() for _, tc := range testCases { diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 0a6903e3d3a7..14d573db5f65 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -54,18 +54,8 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour referenceStore := newProviderReferenceStore() // Create and register the controllers with the manager. - if err := newGatewayClassController(mgr, svr, updateHandler.Writer(), resources); err != nil { - return nil, fmt.Errorf("failed to create gatewayclass controller: %w", err) - } - if err := newGatewayController(mgr, svr, updateHandler.Writer(), resources); err != nil { - return nil, fmt.Errorf("failed to create gateway controller: %w", err) - } - - if err := newHTTPRouteController(mgr, svr, updateHandler.Writer(), resources, referenceStore); err != nil { - return nil, fmt.Errorf("failed to create httproute controller: %w", err) - } - if err := newTLSRouteController(mgr, svr, updateHandler.Writer(), resources, referenceStore); err != nil { - return nil, fmt.Errorf("failed to create tlsroute controller: %w", err) + if err := newGatewayAPIController(mgr, svr, updateHandler.Writer(), resources, referenceStore); err != nil { + return nil, fmt.Errorf("failted to create gatewayapi controller: %w", err) } // Add health check health probes. diff --git a/internal/provider/kubernetes/route.go b/internal/provider/kubernetes/route.go new file mode 100644 index 000000000000..cdc2d5ad4a68 --- /dev/null +++ b/internal/provider/kubernetes/route.go @@ -0,0 +1,421 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package kubernetes + +import ( + "context" + "fmt" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/provider/utils" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// HTTPRoute processing + +// processHTTPRoute processes the HTTPRoute coming from the watcher and further +// processes parent Gateway objects to eventually reconcile GatewayClass. +func (r *gatewayAPIReconciler) processHTTPRoute(obj client.Object) []reconcile.Request { + r.log.Info("processing httproute", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} + + hrkey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + httpRouteParentReferences := []gwapiv1b1.ParentReference{} + routeDeleted := false + + httproute := new(gwapiv1b1.HTTPRoute) + if err := r.client.Get(ctx, hrkey, httproute); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get httproute") + return requests + } + + routeDeleted = true + // Remove the HTTPRoute from watchable map. + if resourceRoute, found := r.resources.HTTPRoutes.Load(hrkey); found { + httpRouteParentReferences = append(httpRouteParentReferences, resourceRoute.Spec.ParentRefs...) + r.resources.HTTPRoutes.Delete(hrkey) + r.log.Info("deleted httproute from watchable map") + } + + // Delete the Namespace and Service from the watchable maps if no other + // routes (TLSRoute or HTTPRoute) exist in the namespace. + found, err := isRoutePresentInNamespace(ctx, r.client, hrkey.Namespace) + if err != nil { + return requests + } + if !found { + r.resources.Namespaces.Delete(hrkey.Namespace) + r.log.Info("deleted namespace from watchable map") + } + + // Delete the Service from the watchable maps if no other + // routes (TLSRoute or HTTPRoute) reference that Service. + routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, hrkey.Namespace, hrkey.Name}) + for svc := range routeServices { + r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindHTTPRoute, hrkey.Namespace, hrkey.Name}, svc) + if !r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(svc) + r.log.Info("deleted service from watchable map", "namespace", svc.Namespace, "name", svc.Name) + } + } + } + + if !routeDeleted { + v, ok := r.resources.HTTPRoutes.Load(hrkey) + // Donot process further if the resource is unchanged. + if ok && httproute.Generation == v.Generation { + return requests + } + if !ok { + r.resources.HTTPRoutes.Store(hrkey, httproute) + r.log.Info("added httproute to watchable map") + } + + // Get the route's namespace from the cache. + if err := r.checkNamespaceForRoute(ctx, hrkey.Namespace); err != nil { + return requests + } + + // Get the route's backendRefs from the cache. Note that a Service is the + // only supported kind. + routeBackendReferences := []gwapiv1b1.BackendRef{} + for i := range httproute.Spec.Rules { + for j := range httproute.Spec.Rules[i].BackendRefs { + ref := httproute.Spec.Rules[i].BackendRefs[j].BackendRef + routeBackendReferences = append(routeBackendReferences, ref) + } + } + if err := r.checkAndValidateRouteBackendRefs(ctx, kindHTTPRoute, hrkey, routeBackendReferences); err != nil { + return requests + } + + httpRouteParentReferences = append(httpRouteParentReferences, httproute.Spec.ParentRefs...) + } + + // Find the parent Gateway objects for httproute. + gateways, err := validateParentRefs(ctx, r.client, hrkey.Namespace, r.classController, httpRouteParentReferences) + if err != nil { + r.log.Info("invalid parentRefs for httproute, bypassing reconciliation", "object", obj) + return requests + } + + // Remove the route from the watchable map since it doesn't reference + // a managed Gateway. + if len(gateways) == 0 { + r.log.Info("httproute doesn't reference any managed gateways") + r.resources.HTTPRoutes.Delete(hrkey) + return requests + } + + for j := range gateways { + requests = append(requests, r.processGateway(gateways[j].DeepCopy())...) + } + + return requests +} + +// addHTTPRouteIndexers adds indexing on HTTPRoute, for Service objects that are +// referenced in HTTPRoute objects via `.spec.rules.backendRefs`. This helps in +// querying for HTTPRoutes that are affected by a particular Service CRUD. +func addHTTPRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1b1.HTTPRoute{}, serviceHTTPRouteIndex, func(rawObj client.Object) []string { + httpRoute := rawObj.(*gwapiv1b1.HTTPRoute) + var backendServices []string + for _, rule := range httpRoute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if string(*backend.Kind) == gatewayapi.KindService { + // If an explicit Service namespace is not provided, use the HTTPRoute namespace to + // lookup the provided Service Name. + backendServices = append(backendServices, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, httpRoute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + return backendServices + }); err != nil { + return err + } + return nil +} + +// TLSRoute processing + +// processTLSRoute processes the TLSRoute coming from the watcher and further +// processes parent Gateway objects to eventually reconcile GatewayClass. +func (r *gatewayAPIReconciler) processTLSRoute(obj client.Object) []reconcile.Request { + r.log.Info("processing tlsroute", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} + + trkey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + tlsRouteParentReferences := []gwapiv1b1.ParentReference{} + routeDeleted := false + + tlsroute := new(gwapiv1a2.TLSRoute) + if err := r.client.Get(ctx, trkey, tlsroute); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get tlsroute") + return requests + } + + routeDeleted = true + // Remove the TLSRoute from watchable map. + if resourceRoute, found := r.resources.TLSRoutes.Load(trkey); found { + tlsRouteParentReferences = append(tlsRouteParentReferences, gatewayapi.UpgradeParentReferences(resourceRoute.Spec.ParentRefs)...) + r.resources.TLSRoutes.Delete(trkey) + r.log.Info("deleted tlsroute from watchable map") + } + + // Delete the Namespace and Service from the watchable maps if no other + // routes (TLSRoute or HTTPRoute) exist in the namespace. + found, err := isRoutePresentInNamespace(ctx, r.client, trkey.Namespace) + if err != nil { + return requests + } + if !found { + r.resources.Namespaces.Delete(trkey.Namespace) + r.log.Info("deleted namespace from watchable map") + } + + // Delete the Service from the watchable maps if no other + // routes (TLSRoute or HTTPRoute) reference that Service. + routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, trkey.Namespace, trkey.Name}) + for svc := range routeServices { + r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, trkey.Namespace, trkey.Name}, svc) + if !r.referenceStore.isServiceReferredByRoutes(svc) { + r.resources.Services.Delete(svc) + r.log.Info("deleted service from watchable map", "namespace", svc.Namespace, "name", svc.Name) + } + } + } + + fmt.Printf("heyoo %v\n", routeDeleted) + + if !routeDeleted { + v, ok := r.resources.TLSRoutes.Load(trkey) + // Donot process further if the resource is unchanged. + if ok && tlsroute.Generation == v.Generation { + return requests + } + + if !ok { + r.resources.TLSRoutes.Store(trkey, tlsroute) + r.log.Info("added tlsroute to watchable map") + } + + // Get the route's namespace from the cache. + if err := r.checkNamespaceForRoute(ctx, trkey.Namespace); err != nil { + return requests + } + + // Get the route's backendRefs from the cache. Note that a Service is the + // only supported kind. + routeBackendReferences := []gwapiv1b1.BackendRef{} + for i := range tlsroute.Spec.Rules { + for j := range tlsroute.Spec.Rules[i].BackendRefs { + ref := gatewayapi.UpgradeBackendRef(tlsroute.Spec.Rules[i].BackendRefs[j]) + routeBackendReferences = append(routeBackendReferences, ref) + } + } + if err := r.checkAndValidateRouteBackendRefs(ctx, kindTLSRoute, trkey, routeBackendReferences); err != nil { + return requests + } + + tlsRouteParentReferences = append(tlsRouteParentReferences, gatewayapi.UpgradeParentReferences(tlsroute.Spec.ParentRefs)...) + } + + // Find the parent Gateway objects for tlsroute. + gateways, err := validateParentRefs(ctx, r.client, trkey.Namespace, r.classController, tlsRouteParentReferences) + if err != nil { + r.log.Info("invalid parentRefs for tlsroute, bypassing reconciliation", "object", obj) + return requests + } + + // Remove the route from the watchable map since it doesn't reference + // a managed Gateway. + if len(gateways) == 0 { + r.log.Info("tlsroute doesn't reference any managed gateways") + r.resources.TLSRoutes.Delete(trkey) + return requests + } + + for j := range gateways { + requests = append(requests, r.processGateway(gateways[j].DeepCopy())...) + } + + return requests +} + +// addTLSRouteIndexers adds indexing on TLSRoute, for Service objects that are +// referenced in TLSRoute objects via `.spec.rules.backendRefs`. This helps in +// querying for TLSRoutes that are affected by a particular Service CRUD. +func addTLSRouteIndexers(ctx context.Context, mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a2.TLSRoute{}, serviceTLSRouteIndex, func(rawObj client.Object) []string { + tlsRoute := rawObj.(*gwapiv1a2.TLSRoute) + var backendServices []string + for _, rule := range tlsRoute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if string(*backend.Kind) == gatewayapi.KindService { + // If an explicit Service namespace is not provided, use the TLSRoute namespace to + // lookup the provided Service Name. + backendServices = append(backendServices, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, tlsRoute.Namespace), + Name: string(backend.Name), + }.String(), + ) + } + } + } + return backendServices + }); err != nil { + return err + } + return nil +} + +// Common Route processing functions + +// processService processes the Service coming from the watcher and further +// processes parent Route objects to eventually reconcile GatewayClass. +func (r *gatewayAPIReconciler) processService(obj client.Object) []reconcile.Request { + r.log.Info("processing service", "namespace", obj.GetNamespace(), "name", obj.GetName()) + ctx := context.Background() + requests := []reconcile.Request{} + + svckey := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + serviceDeleted := false + + service := new(corev1.Service) + if err := r.client.Get(ctx, svckey, service); err != nil { + if !kerrors.IsNotFound(err) { + r.log.Error(err, "failed to get service") + return requests + } + + serviceDeleted = true + // Remove the Service from watchable map. + r.resources.Services.Delete(svckey) + } + + if !serviceDeleted { + // Store the Service in watchable map. + r.resources.Services.Store(svckey, service.DeepCopy()) + } + + // Find the HTTPRoutes that reference this Service. + httpRouteList := &gwapiv1b1.HTTPRouteList{} + if err := r.client.List(context.Background(), httpRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(serviceHTTPRouteIndex, utils.NamespacedName(obj).String()), + }); err != nil { + return []reconcile.Request{} + } + + // Find the TLSRoutes that reference this Service. + tlsRouteList := &gwapiv1a2.TLSRouteList{} + if err := r.client.List(context.Background(), tlsRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(serviceTLSRouteIndex, utils.NamespacedName(obj).String()), + }); err != nil { + return []reconcile.Request{} + } + + for _, h := range httpRouteList.Items { + requests = append(requests, r.processHTTPRoute(h.DeepCopy())...) + } + for _, t := range tlsRouteList.Items { + requests = append(requests, r.processTLSRoute(t.DeepCopy())...) + } + + return requests +} + +// checkNamespaceForRoute +func (r *gatewayAPIReconciler) checkNamespaceForRoute(ctx context.Context, name string) error { + nsKey := types.NamespacedName{Name: name} + ns := new(corev1.Namespace) + if err := r.client.Get(ctx, nsKey, ns); err != nil { + if kerrors.IsNotFound(err) { + // The route's namespace doesn't exist in the cache, so remove it from + // the namespace watchable map if it exists. + if _, ok := r.resources.Namespaces.Load(nsKey.Name); ok { + r.resources.Namespaces.Delete(nsKey.Name) + r.log.Info("deleted namespace from watchable map") + } + } + return fmt.Errorf("failed to get namespace %s", nsKey.Name) + } + + // The route's namespace exists, so add it to the watchable map. + r.resources.Namespaces.Store(nsKey.Name, ns) + r.log.Info("added namespace to watchable map") + return nil +} + +// checkAndValidateRouteBackendRefs +func (r *gatewayAPIReconciler) checkAndValidateRouteBackendRefs(ctx context.Context, routeKind string, routekey types.NamespacedName, backendReferences []gwapiv1b1.BackendRef) error { + for j := range backendReferences { + ref := backendReferences[j] + if err := validateBackendRef(&ref); err != nil { + return fmt.Errorf("invalid backendRef: %w", err) + } + + // The backendRef is valid, so get the referenced service from the cache. + svcKey := types.NamespacedName{Namespace: routekey.Namespace, Name: string(ref.Name)} + svc := new(corev1.Service) + if err := r.client.Get(ctx, svcKey, svc); err != nil { + if kerrors.IsNotFound(err) { + // The ref's service doesn't exist in the cache, so remove it from + // the watchable map if it exists. + if _, ok := r.resources.Services.Load(svcKey); ok { + r.resources.Services.Delete(svcKey) + r.referenceStore.removeRouteToServicesMapping( + ObjectKindNamespacedName{routeKind, routekey.Namespace, routekey.Name}, + svcKey, + ) + r.log.Info("deleted service from watchable map") + } + } + return fmt.Errorf("failed to get service %s/%s", + svcKey.Namespace, svcKey.Name) + } + + // The backendRef Service exists, so add it to the watchable map. + r.resources.Services.Store(svcKey, svc) + r.referenceStore.updateRouteToServicesMapping( + ObjectKindNamespacedName{routeKind, routekey.Namespace, routekey.Name}, + svcKey, + ) + r.log.Info("added service to watchable map") + } + return nil +} diff --git a/internal/provider/kubernetes/tlsroute.go b/internal/provider/kubernetes/tlsroute.go deleted file mode 100644 index 9cd3db39e9c4..000000000000 --- a/internal/provider/kubernetes/tlsroute.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -// This file contains code derived from Contour, -// https://github.com/projectcontour/contour -// from the source file -// https://github.com/projectcontour/contour/blob/main/internal/controller/tlsroute.go -// and is provided here subject to the following: -// Copyright Project Contour Authors -// SPDX-License-Identifier: Apache-2.0 - -package kubernetes - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/gatewayapi" - "github.com/envoyproxy/gateway/internal/message" - "github.com/envoyproxy/gateway/internal/provider/utils" - "github.com/envoyproxy/gateway/internal/status" -) - -const ( - kindTLSRoute = "TLSRoute" - - serviceTLSRouteIndex = "serviceTLSRouteBackendRef" -) - -type tlsRouteReconciler struct { - client client.Client - log logr.Logger - statusUpdater status.Updater - classController gwapiv1b1.GatewayController - - resources *message.ProviderResources - referenceStore *providerReferenceStore -} - -// newTLSRouteController creates the tlsroute controller from mgr. The controller will be pre-configured -// to watch for TLSRoute objects across all namespaces. -func newTLSRouteController(mgr manager.Manager, cfg *config.Server, su status.Updater, resources *message.ProviderResources, referenceStore *providerReferenceStore) error { - r := &tlsRouteReconciler{ - client: mgr.GetClient(), - log: cfg.Logger, - classController: gwapiv1b1.GatewayController(cfg.EnvoyGateway.Gateway.ControllerName), - statusUpdater: su, - resources: resources, - referenceStore: referenceStore, - } - - c, err := controller.New("tlsroute", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - r.log.Info("created tlsroute controller") - - if err := c.Watch( - &source.Kind{Type: &gwapiv1a2.TLSRoute{}}, - &handler.EnqueueRequestForObject{}, - ); err != nil { - return err - } - - // Subscribe to status updates - go r.subscribeAndUpdateStatus(context.Background()) - - // Add indexing on TLSRoute, for Service objects that are referenced in TLSRoute objects - // via `.spec.rules.backendRefs`. This helps in querying for TLSRoutes that are affected by - // a particular Service CRUD. - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &gwapiv1a2.TLSRoute{}, serviceTLSRouteIndex, func(rawObj client.Object) []string { - tlsRoute := rawObj.(*gwapiv1a2.TLSRoute) - var backendServices []string - for _, rule := range tlsRoute.Spec.Rules { - for _, backend := range rule.BackendRefs { - if string(*backend.Kind) == gatewayapi.KindService { - // If an explicit Service namespace is not provided, use the TLSRoute namespace to - // lookup the provided Service Name. - backendServices = append(backendServices, - types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, tlsRoute.Namespace), - Name: string(backend.Name), - }.String(), - ) - } - } - } - return backendServices - }); err != nil { - return err - } - - // Watch Gateway CRUDs and reconcile affected TLSRoutes. - if err := c.Watch( - &source.Kind{Type: &gwapiv1b1.Gateway{}}, - handler.EnqueueRequestsFromMapFunc(r.getTLSRoutesForGateway), - ); err != nil { - return err - } - - // Watch Service CRUDs and reconcile affected TLSRoutes. - if err := c.Watch( - &source.Kind{Type: &corev1.Service{}}, - handler.EnqueueRequestsFromMapFunc(r.getTLSRoutesForService), - ); err != nil { - return err - } - - r.log.Info("watching tlsroute objects") - return nil -} - -// getTLSRoutesForGateway uses a Gateway obj to fetch TLSRoutes, iterating -// through them and creating a reconciliation request for each valid TLSRoute -// that references obj. -func (r *tlsRouteReconciler) getTLSRoutesForGateway(obj client.Object) []reconcile.Request { - ctx := context.Background() - - gw, ok := obj.(*gwapiv1b1.Gateway) - if !ok { - r.log.Info("unexpected object type, bypassing reconciliation", "object", obj) - return []reconcile.Request{} - } - - routes := &gwapiv1a2.TLSRouteList{} - if err := r.client.List(ctx, routes); err != nil { - return []reconcile.Request{} - } - - requests := []reconcile.Request{} - for i := range routes.Items { - route := routes.Items[i] - gateways, err := validateParentRefs(ctx, r.client, route.Namespace, r.classController, gatewayapi.UpgradeParentReferences(route.Spec.ParentRefs)) - if err != nil { - r.log.Info("invalid parentRefs for tlsroute, bypassing reconciliation", "object", obj) - continue - } - for j := range gateways { - if gateways[j].Namespace == gw.Namespace && gateways[j].Name == gw.Name { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: route.Namespace, - Name: route.Name, - }, - } - requests = append(requests, req) - break - } - } - } - - return requests -} - -// getTLSRoutesForService uses a Service obj to fetch TLSRoutes that references -// the Service using `.spec.rules.backendRefs`. The affected TLSRoutes are then -// pushed for reconciliation. -func (r *tlsRouteReconciler) getTLSRoutesForService(obj client.Object) []reconcile.Request { - affectedTLSRouteList := &gwapiv1a2.TLSRouteList{} - - if err := r.client.List(context.Background(), affectedTLSRouteList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(serviceTLSRouteIndex, utils.NamespacedName(obj).String()), - }); err != nil { - return []reconcile.Request{} - } - - requests := make([]reconcile.Request, len(affectedTLSRouteList.Items)) - for i, item := range affectedTLSRouteList.Items { - requests[i] = reconcile.Request{ - NamespacedName: utils.NamespacedName(item.DeepCopy()), - } - } - - return requests -} - -func (r *tlsRouteReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log := r.log.WithValues("namespace", request.Namespace, "name", request.Name) - - log.Info("reconciling tlsroute") - - // Fetch all TLSRoutes from the cache. - routeList := &gwapiv1a2.TLSRouteList{} - if err := r.client.List(ctx, routeList); err != nil { - return reconcile.Result{}, fmt.Errorf("error listing tlsroutes") - } - - found := false - for i := range routeList.Items { - // See if this route from the list matched the reconciled route. - route := routeList.Items[i] - routeKey := utils.NamespacedName(&route) - if routeKey == request.NamespacedName { - found = true - } - - // Store the tlsroute in the resource map. - r.resources.TLSRoutes.Store(routeKey, &route) - log.Info("added tlsroute to resource map") - - // Get the route's namespace from the cache. - nsKey := types.NamespacedName{Name: route.Namespace} - ns := new(corev1.Namespace) - if err := r.client.Get(ctx, nsKey, ns); err != nil { - if errors.IsNotFound(err) { - // The route's namespace doesn't exist in the cache, so remove it from - // the namespace resource map if it exists. - if _, ok := r.resources.Namespaces.Load(nsKey.Name); ok { - r.resources.Namespaces.Delete(nsKey.Name) - log.Info("deleted namespace from resource map") - } - } - return reconcile.Result{}, fmt.Errorf("failed to get namespace %s", nsKey.Name) - } - - // The route's namespace exists, so add it to the resource map. - r.resources.Namespaces.Store(nsKey.Name, ns) - log.Info("added namespace to resource map") - - // Get the route's backendRefs from the cache. Note that a Service is the - // only supported kind. - for i := range route.Spec.Rules { - for j := range route.Spec.Rules[i].BackendRefs { - ref := route.Spec.Rules[i].BackendRefs[j] - if err := validateTLSRouteBackendRef(&ref); err != nil { - return reconcile.Result{}, fmt.Errorf("invalid backendRef: %w", err) - } - - // The backendRef is valid, so get the referenced service from the cache. - svcKey := types.NamespacedName{Namespace: route.Namespace, Name: string(ref.Name)} - svc := new(corev1.Service) - if err := r.client.Get(ctx, svcKey, svc); err != nil { - if errors.IsNotFound(err) { - // The ref's service doesn't exist in the cache, so remove it from - // the resource map if it exists. - if _, ok := r.resources.Services.Load(svcKey); ok { - r.resources.Services.Delete(svcKey) - r.referenceStore.removeRouteToServicesMapping( - ObjectKindNamespacedName{kindTLSRoute, route.Namespace, route.Name}, - svcKey, - ) - log.Info("deleted service from resource map") - } - } - return reconcile.Result{}, fmt.Errorf("failed to get service %s/%s", - svcKey.Namespace, svcKey.Name) - } - - // The backendRef Service exists, so add it to the resource map. - r.resources.Services.Store(svcKey, svc) - r.referenceStore.updateRouteToServicesMapping( - ObjectKindNamespacedName{kindTLSRoute, route.Namespace, route.Name}, - svcKey, - ) - log.Info("added service to resource map") - } - } - } - - if !found { - // Delete the tlsroute from the resource map. - r.resources.TLSRoutes.Delete(request.NamespacedName) - log.Info("deleted tlsroute from resource map") - - // Delete the Namespace from the resource maps if no other - // routes (TLSRoute/HTTPRoute) exist in the namespace. - if found, err := isRoutePresentInNamespace(ctx, r.client, request.NamespacedName.Namespace); err != nil { - return reconcile.Result{}, err - } else if !found { - r.resources.Namespaces.Delete(request.Namespace) - log.Info("deleted namespace from resource map") - } - - // Delete the Service from the resource maps if no other - // routes (TLSRoute or HTTPRoute) reference that Service. - routeServices := r.referenceStore.getRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}) - for svc := range routeServices { - r.referenceStore.removeRouteToServicesMapping(ObjectKindNamespacedName{kindTLSRoute, request.Namespace, request.Name}, svc) - if !r.referenceStore.isServiceReferredByRoutes(svc) { - r.resources.Services.Delete(svc) - log.Info("deleted service from resource map", "namespace", svc.Namespace, "name", svc.Name) - } - } - } - - log.Info("reconciled tlsroute") - - return reconcile.Result{}, nil -} - -// validateTLSRouteBackendRef validates that ref is a reference to a local Service. -func validateTLSRouteBackendRef(ref *gwapiv1a2.BackendRef) error { - switch { - case ref == nil: - return nil - case ref.Group != nil && *ref.Group != corev1.GroupName: - return fmt.Errorf("invalid group; must be nil or empty string") - case ref.Kind != nil && *ref.Kind != gatewayapi.KindService: - return fmt.Errorf("invalid kind %q; must be %q", - *ref.BackendObjectReference.Kind, gatewayapi.KindService) - case ref.Namespace != nil: - return fmt.Errorf("invalid namespace; must be nil") - } - - return nil -} - -// subscribeAndUpdateStatus subscribes to tlsroute status updates and writes it into the -// Kubernetes API Server -func (r *tlsRouteReconciler) subscribeAndUpdateStatus(ctx context.Context) { - // Subscribe to resources - message.HandleSubscription(r.resources.TLSRouteStatuses.Subscribe(ctx), - func(update message.Update[types.NamespacedName, *gwapiv1a2.TLSRoute]) { - // skip delete updates. - if update.Delete { - return - } - key := update.Key - val := update.Value - r.statusUpdater.Send(status.Update{ - NamespacedName: key, - Resource: new(gwapiv1a2.TLSRoute), - Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*gwapiv1a2.TLSRoute) - if !ok { - panic(fmt.Sprintf("unsupported object type %T", obj)) - } - tCopy := t.DeepCopy() - tCopy.Status.Parents = val.Status.Parents - return tCopy - }), - }) - }, - ) - r.log.Info("status subscriber shutting down") -}