[DONT MERGE] Add logic to get backoff metrics

internal/backoff/backoff.go

@@ -41,15 +41,29 @@ type backoff struct {
 	maxRetryCount         int
 	backoffTimeLimit      time.Duration
 	errLog                bool
+	metricsEnabled        bool
+	metrics               sync.Map
 }
 
 // Backoff represents an interface to handle backoff operation.
 type Backoff interface {
 	Do(context.Context, func(ctx context.Context) (interface{}, bool, error)) (interface{}, error)
+	Metrics(ctx context.Context) map[string]int
 	Close()
 }
 
-const traceTag = "vald/internal/backoff/Backoff.Do/retry"
+type contextKey string
+
+const (
+	traceTag = "vald/internal/backoff/Backoff.Do/retry"
+
+	serviceContextKey contextKey = "service"
+)
+
+// WithServiceContext returns a copy of parent in which the value associated with key (serviceContextKey).
+func WithServiceContext(ctx context.Context, svc string) context.Context {
+	return context.WithValue(ctx, serviceContextKey, svc)
+}
 
 // New creates the new backoff with option.
 func New(opts ...Option) Backoff {
@@ -110,6 +124,11 @@ func (b *backoff) Do(ctx context.Context, f func(ctx context.Context) (val inter
 				}()
 				return f(ssctx)
 			}()
+
+			if svc := ctx.Value(serviceContextKey); svc != nil && b.metricsEnabled {
+				b.metrics.Store(svc.(string), cnt+1)
+			}
+
 			if !ret {
 				return res, err
 			}
@@ -149,6 +168,15 @@ func (b *backoff) addJitter(dur float64) float64 {
 	return dur + float64(rand.LimitedUint32(uint64(hd))) - hd
 }
 
+func (b *backoff) Metrics(_ context.Context) (m map[string]int) {
+	b.metrics.Range(func(key, value any) bool {
+		b.metrics.Delete(key)
+		m[key.(string)] = value.(int)
+		return true
+	})
+	return m
+}
+
 // Close wait for the backoff process to complete.
 func (b *backoff) Close() {
 	b.wg.Wait()

internal/backoff/option.go

@@ -34,6 +34,7 @@ var defaultOptions = []Option{
 	WithBackOffFactor(1.5),
 	WithRetryCount(50),
 	WithEnableErrorLog(),
+	WithDisableMetrics(),
 }
 
 // WithInitialDuration returns the option to set the initial duration of backoff.
@@ -113,3 +114,17 @@ func WithDisableErrorLog() Option {
 		b.errLog = false
 	}
 }
+
+// WithEnableMetrics returns the option to set the enable for backoff metrics.
+func WithEnableMetrics() Option {
+	return func(b *backoff) {
+		b.metricsEnabled = true
+	}
+}
+
+// WithDisableMetrics returns the option to set the disable for backoff metrics.
+func WithDisableMetrics() Option {
+	return func(b *backoff) {
+		b.metricsEnabled = false
+	}
+}

internal/net/http/client/client_test.go

@@ -49,7 +49,7 @@ var (
 		comparator.AllowUnexported(http.Transport{}),
 		comparator.IgnoreFields(http.Transport{}, "idleLRU", "altProto", "TLSNextProto"),
 		comparator.Exporter(func(t reflect.Type) bool {
-			if t.Name() == "ert" || t.Name() == "backoff" {
+			if t.Name() == "ert" || t.Name() == "backoff" || t.Name() == "Map" {
 				return true
 			}
 			return false

internal/net/http/transport/roundtrip_mock.go

@@ -29,14 +29,19 @@ func (rm *roundTripMock) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 type backoffMock struct {
-	DoFunc    func(context.Context, func(context.Context) (interface{}, bool, error)) (interface{}, error)
-	CloseFunc func()
+	DoFunc      func(context.Context, func(context.Context) (interface{}, bool, error)) (interface{}, error)
+	MetricsFunc func(ctx context.Context) map[string]int
+	CloseFunc   func()
 }
 
 func (bm *backoffMock) Do(ctx context.Context, fn func(context.Context) (interface{}, bool, error)) (interface{}, error) {
 	return bm.DoFunc(ctx, fn)
 }
 
+func (bm *backoffMock) Metrics(ctx context.Context) map[string]int {
+	return bm.MetricsFunc(ctx)
+}
+
 func (bm *backoffMock) Close() {
 	bm.CloseFunc()
 }

internal/observability/metrics/backoff/backoff.go

@@ -0,0 +1,59 @@
+package backoff
+
+import (
+	"context"
+
+	"github.com/vdaas/vald/internal/backoff"
+	"github.com/vdaas/vald/internal/observability/metrics"
+)
+
+var (
+	serviceNameKey = metrics.MustNewKey("grpc_service")
+)
+
+type backoffMetrics struct {
+	bo         backoff.Backoff
+	retryCount metrics.Int64Measure
+}
+
+func New(bo backoff.Backoff) metrics.Metric {
+	return &backoffMetrics{
+		bo: bo,
+		retryCount: *metrics.Int64(
+			metrics.ValdOrg+"/grpc/backoff/retry_count",
+			"Backoff retry count",
+			metrics.UnitDimensionless),
+	}
+}
+
+func (*backoffMetrics) Measurement(_ context.Context) ([]metrics.Measurement, error) {
+	return []metrics.Measurement{}, nil
+}
+
+func (bm *backoffMetrics) MeasurementWithTags(ctx context.Context) (mts []metrics.MeasurementWithTags, err error) {
+	ms := bm.bo.Metrics(ctx)
+	mts = make([]metrics.MeasurementWithTags, 0, len(ms))
+	for svc, cnt := range ms {
+		mts = append(mts, metrics.MeasurementWithTags{
+			Measurement: bm.retryCount.M(int64(cnt)),
+			Tags: map[metrics.Key]string{
+				serviceNameKey: svc,
+			},
+		})
+	}
+	return mts, nil
+}
+
+func (bm *backoffMetrics) View() []*metrics.View {
+	return []*metrics.View{
+		{
+			Name:        "backoff_retry_count",
+			Description: bm.retryCount.Description(),
+			Measure:     &bm.retryCount,
+			TagKeys: []metrics.Key{
+				serviceNameKey,
+			},
+			Aggregation: metrics.LastValue(),
+		},
+	}
+}

internal/runner/runner_race_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/vdaas/vald/internal/test/goleak"
 )
 
-func TestDo(t *testing.T) {
+func TestDo_for_race(t *testing.T) {
 	type args struct {
 		ctx  context.Context
 		opts []Option