Ignore endpoint events after retrying couple of times

If the endpoint handlers cannot recover from a failure, even after retrying 20 times, it's best to disregard the event. This prevents the logs from being flooded with repetitive errors. Signed-off-by: Sridhar Gaddam <[email protected]>
submariner-io · Aug 28, 2023 · 740b0b4 · 740b0b4
1 parent 2a51058
commit 740b0b4
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 3 deletions.
diff --git a/pkg/event/controller/controller.go b/pkg/event/controller/controller.go
@@ -54,6 +54,10 @@ type Controller struct {
 	isGatewayNode bool
 }
 
+// If the handler cannot recover from a failure, even after retrying for maximum requeue attempts,
+// it's best to disregard the event. This prevents the logs from being flooded with repetitive errors.
+const maxRequeues = 20
+
 type Config struct {
 	// Registry is the event handler registry where controller events will be sent.
 	Registry *event.Registry

diff --git a/pkg/event/controller/endpoint_created.go b/pkg/event/controller/endpoint_created.go
@@ -23,11 +23,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-func (c *Controller) handleCreatedEndpoint(obj runtime.Object, _ int) bool {
+func (c *Controller) handleCreatedEndpoint(obj runtime.Object, requeueCount int) bool {
 	var err error
 
 	endpoint := obj.(*smv1.Endpoint)
 
+	if requeueCount > maxRequeues {
+		logger.Errorf(nil, "Ignoring create event for endpoint %q, as its requeued for more than %d times",
+			endpoint.Spec.ClusterID, maxRequeues)
+		return false
+	}
+
 	if endpoint.Spec.ClusterID != c.env.ClusterID {
 		err = c.handleCreatedRemoteEndpoint(endpoint)
 	} else {

diff --git a/pkg/event/controller/endpoint_removed.go b/pkg/event/controller/endpoint_removed.go
@@ -23,9 +23,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-func (c *Controller) handleRemovedEndpoint(obj runtime.Object, _ int) bool {
+func (c *Controller) handleRemovedEndpoint(obj runtime.Object, requeueCount int) bool {
 	endpoint := obj.(*smv1.Endpoint)
 
+	if requeueCount > maxRequeues {
+		logger.Errorf(nil, "Ignoring delete event for endpoint %q, as its requeued for more than %d times",
+			endpoint.Spec.ClusterID, maxRequeues)
+		return false
+	}
+
 	var err error
 	if endpoint.Spec.ClusterID != c.env.ClusterID {
 		err = c.handleRemovedRemoteEndpoint(endpoint)

diff --git a/pkg/event/controller/endpoint_updated.go b/pkg/event/controller/endpoint_updated.go
@@ -23,9 +23,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-func (c *Controller) handleUpdatedEndpoint(obj runtime.Object, _ int) bool {
+func (c *Controller) handleUpdatedEndpoint(obj runtime.Object, requeueCount int) bool {
 	endpoint := obj.(*smv1.Endpoint)
 
+	if requeueCount > maxRequeues {
+		logger.Errorf(nil, "Ignoring update event for endpoint %q, as its requeued for more than %d times",
+			endpoint.Spec.ClusterID, maxRequeues)
+		return false
+	}
+
 	var err error
 	if endpoint.Spec.ClusterID != c.env.ClusterID {
 		err = c.handleUpdatedRemoteEndpoint(endpoint)