tetra: implement exponential backoff for RPC calls

This implement a helper that can be used to add exponential backoff to a
RPC call. This can't be used with existing CliRunErr and CliRun as of
now since they don't return the error out of the closure.

Signed-off-by: Mahe Tardy <[email protected]>

cmd/tetra/common/client.go

@@ -11,8 +11,12 @@ import (
 	"time"
 
 	"github.com/cilium/tetragon/api/v1/tetragon"
+	"github.com/cilium/tetragon/pkg/logger"
+	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
 )
 
 func CliRunErr(fn func(ctx context.Context, cli tetragon.FineGuidanceSensorsClient), fnErr func(err error)) {
@@ -68,3 +72,33 @@ func NewClient(ctx context.Context, address string, timeout time.Duration) (*Cli
 
 	return c, nil
 }
+
+// ExpBackoff retries calling the function f as many time as specified by the
+// global var Retries retrieved from flag. You must pass the context of the
+// calling client to also respect interrupt and timeout.
+func ExpBackoff[T any](ctx context.Context, f func() (T, error)) (res T, err error) {
+	backoff := time.Second
+	for attempt := range Retries {
+		res, err = f()
+		if err != nil {
+			if status.Code(err) != codes.Unavailable {
+				break
+			}
+
+			logger.GetLogger().WithFields(logrus.Fields{
+				"attempt": attempt + 1,
+				"backoff": backoff,
+			}).WithError(err).Error("RPC call failed, retrying...")
+
+			select {
+			case <-time.After(backoff):
+			case <-ctx.Done():
+				break
+			}
+
+			backoff *= 2
+			continue
+		}
+	}
+	return
+}

cmd/tetra/tracingpolicy/tracingpolicy.go

@@ -38,8 +38,10 @@ func New() *cobra.Command {
 				return fmt.Errorf("failed to read yaml file %s: %w", args[0], err)
 			}
 
-			_, err = c.Client.AddTracingPolicy(c.Ctx, &tetragon.AddTracingPolicyRequest{
-				Yaml: string(yamlb),
+			_, err = common.ExpBackoff(c.Ctx, func() (any, error) {
+				return c.Client.AddTracingPolicy(c.Ctx, &tetragon.AddTracingPolicyRequest{
+					Yaml: string(yamlb),
+				})
 			})
 			if err != nil {
 				return fmt.Errorf("failed to add tracing policy: %w", err)
@@ -62,9 +64,11 @@ func New() *cobra.Command {
 			}
 			defer c.Close()
 
-			_, err = c.Client.DeleteTracingPolicy(c.Ctx, &tetragon.DeleteTracingPolicyRequest{
-				Name:      args[0],
-				Namespace: tpDelNamespaceFlag,
+			_, err = common.ExpBackoff(c.Ctx, func() (any, error) {
+				return c.Client.DeleteTracingPolicy(c.Ctx, &tetragon.DeleteTracingPolicyRequest{
+					Name:      args[0],
+					Namespace: tpDelNamespaceFlag,
+				})
 			})
 			if err != nil {
 				return fmt.Errorf("failed to delete tracing policy: %w", err)
@@ -88,9 +92,11 @@ func New() *cobra.Command {
 			}
 			defer c.Close()
 
-			_, err = c.Client.EnableTracingPolicy(c.Ctx, &tetragon.EnableTracingPolicyRequest{
-				Name:      args[0],
-				Namespace: tpEnableNamespaceFlag,
+			_, err = common.ExpBackoff(c.Ctx, func() (any, error) {
+				return c.Client.EnableTracingPolicy(c.Ctx, &tetragon.EnableTracingPolicyRequest{
+					Name:      args[0],
+					Namespace: tpEnableNamespaceFlag,
+				})
 			})
 			if err != nil {
 				return fmt.Errorf("failed to enable tracing policy: %w", err)
@@ -114,11 +120,12 @@ func New() *cobra.Command {
 			}
 			defer c.Close()
 
-			_, err = c.Client.DisableTracingPolicy(c.Ctx, &tetragon.DisableTracingPolicyRequest{
-				Name:      args[0],
-				Namespace: tpDisableNamespaceFlag,
+			_, err = common.ExpBackoff(c.Ctx, func() (any, error) {
+				return c.Client.DisableTracingPolicy(c.Ctx, &tetragon.DisableTracingPolicyRequest{
+					Name:      args[0],
+					Namespace: tpDisableNamespaceFlag,
+				})
 			})
-
 			if err != nil {
 				return fmt.Errorf("failed to disable tracing policy: %w", err)
 			}
@@ -148,7 +155,9 @@ func New() *cobra.Command {
 			}
 			defer c.Close()
 
-			res, err := c.Client.ListTracingPolicies(c.Ctx, &tetragon.ListTracingPoliciesRequest{})
+			res, err := common.ExpBackoff(c.Ctx, func() (*tetragon.ListTracingPoliciesResponse, error) {
+				return c.Client.ListTracingPolicies(c.Ctx, &tetragon.ListTracingPoliciesRequest{})
+			})
 			if err != nil || res == nil {
 				return fmt.Errorf("failed to list tracing policies: %w", err)
 			}