[DONT MERGE] Add logic to get backoff metrics

charts/vald-helm-operator/crds/valdrelease.yaml

@@ -1224,6 +1224,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -1284,6 +1286,8 @@ spec:
                                   type: string
                                 enable_error_log:
                                   type: boolean
+                                enable_metrics:
+                                  type: boolean
                                 initial_duration:
                                   type: string
                                 jitter_limit:
@@ -2152,6 +2156,8 @@ spec:
                                   type: string
                                 enable_error_log:
                                   type: boolean
+                                enable_metrics:
+                                  type: boolean
                                 initial_duration:
                                   type: string
                                 jitter_limit:
@@ -4193,6 +4199,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -4357,6 +4365,8 @@ spec:
                                       type: string
                                     enable_error_log:
                                       type: boolean
+                                    enable_metrics:
+                                      type: boolean
                                     initial_duration:
                                       type: string
                                     jitter_limit:
@@ -4516,6 +4526,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -5648,6 +5660,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -5804,6 +5818,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -6956,6 +6972,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:
@@ -7112,6 +7130,8 @@ spec:
                                           type: string
                                         enable_error_log:
                                           type: boolean
+                                        enable_metrics:
+                                          type: boolean
                                         initial_duration:
                                           type: string
                                         jitter_limit:

charts/vald/values.yaml

@@ -671,6 +671,9 @@ defaults:
         # @schema {"name": "defaults.grpc.client.backoff.enable_error_log", "type": "boolean"}
         # defaults.grpc.client.backoff.enable_error_log -- gRPC client backoff log enabled
         enable_error_log: true
+        # @schema {"name": "defaults.grpc.client.backoff.enable_metrics", "type": "boolean"}
+        # defaults.grpc.client.backoff.enable_metrics -- gRPC client backoff metrics enabled
+        enable_metrics: false
       # @schema {"name": "defaults.grpc.client.call_option", "type": "object"}
       call_option:
         # @schema {"name": "defaults.grpc.client.wait_for_ready", "type": "boolean"}

internal/backoff/backoff.go

@@ -41,11 +41,16 @@ type backoff struct {
 	maxRetryCount         int
 	backoffTimeLimit      time.Duration
 	errLog                bool
+	metricsEnabled        bool
+
+	mu      sync.RWMutex
+	metrics map[string]int64
 }
 
 // Backoff represents an interface to handle backoff operation.
 type Backoff interface {
 	Do(context.Context, func(ctx context.Context) (interface{}, bool, error)) (interface{}, error)
+	Metrics(ctx context.Context) map[string]int64
 	Close()
 }
 
@@ -62,6 +67,7 @@ func New(opts ...Option) Backoff {
 	}
 	b.durationLimit = b.maxDuration / b.backoffFactor
 	b.jittedInitialDuration = b.addJitter(b.initialDuration)
+	b.metrics = make(map[string]int64)
 
 	return b
 }
@@ -86,6 +92,7 @@ func (b *backoff) Do(ctx context.Context, f func(ctx context.Context) (val inter
 
 	dur := b.initialDuration
 	jdur := b.jittedInitialDuration
+	name := FromBackoffName(ctx)
 
 	dctx, cancel := context.WithDeadline(sctx, time.Now().Add(b.backoffTimeLimit))
 	defer cancel()
@@ -119,6 +126,12 @@ func (b *backoff) Do(ctx context.Context, f func(ctx context.Context) (val inter
 			if b.errLog {
 				log.Error(err)
 			}
+			// e.g. name = v1.vald.Exists/10.0.0.0 ...etc
+			if len(name) != 0 && b.metricsEnabled {
+				b.mu.Lock()
+				b.metrics[name] += 1
+				b.mu.Unlock()
+			}
 			timer.Reset(time.Duration(jdur))
 			select {
 			case <-dctx.Done():
@@ -149,6 +162,20 @@ func (b *backoff) addJitter(dur float64) float64 {
 	return dur + float64(rand.LimitedUint32(uint64(hd))) - hd
 }
 
+func (b *backoff) Metrics(_ context.Context) map[string]int64 {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	if len(b.metrics) == 0 {
+		return nil
+	}
+
+	m := make(map[string]int64, len(b.metrics))
+	for name, cnt := range b.metrics {
+		m[name] = cnt
+	}
+	return m
+}
+
 // Close wait for the backoff process to complete.
 func (b *backoff) Close() {
 	b.wg.Wait()

internal/backoff/backoff_test.go

@@ -61,6 +61,7 @@ func TestNew(t *testing.T) {
 				maxRetryCount:    50,
 				errLog:           true,
 				durationLimit:    float64(time.Hour) / 1.1,
+				metrics:          make(map[string]int64),
 			},
 			checkFunc: func(got *backoff, want *backoff) error {
 				got.jittedInitialDuration, want.jittedInitialDuration = 1, 1

internal/backoff/context.go

@@ -0,0 +1,35 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package backoff
+
+import "context"
+
+type contextKey string
+
+const backoffNameContextKey contextKey = "backoff_name"
+
+// WithBackoffName returns a copy of parent in which the method associated with key (backoffNameContextKey).
+func WithBackoffName(ctx context.Context, name string) context.Context {
+	return context.WithValue(ctx, backoffNameContextKey, name)
+}
+
+// FromBackoffName returns the value associated with this context for key (backoffNameContextKey).
+func FromBackoffName(ctx context.Context) string {
+	if val := ctx.Value(backoffNameContextKey); val != nil {
+		return val.(string)
+	}
+	return ""
+}

internal/backoff/option.go

@@ -34,6 +34,7 @@ var defaultOptions = []Option{
 	WithBackOffFactor(1.5),
 	WithRetryCount(50),
 	WithEnableErrorLog(),
+	WithDisableMetrics(),
 }
 
 // WithInitialDuration returns the option to set the initial duration of backoff.
@@ -113,3 +114,17 @@ func WithDisableErrorLog() Option {
 		b.errLog = false
 	}
 }
+
+// WithEnableMetrics returns the option to set the enable for backoff metrics.
+func WithEnableMetrics() Option {
+	return func(b *backoff) {
+		b.metricsEnabled = true
+	}
+}
+
+// WithDisableMetrics returns the option to set the disable for backoff metrics.
+func WithDisableMetrics() Option {
+	return func(b *backoff) {
+		b.metricsEnabled = false
+	}
+}

internal/client/v1/client/discoverer/discover.go

@@ -218,6 +218,7 @@ func (c *client) dnsDiscovery(ctx context.Context, ech chan<- error) (addrs []st
 }
 
 func (c *client) discover(ctx context.Context, ech chan<- error) (err error) {
+	ctx = grpc.WithGRPCMethod(ctx, "discoverer.v1.Discoverer/Nodes")
 	if c.dscClient == nil || (c.autoconn && c.client == nil) {
 		return errors.ErrGRPCClientNotFound
 	}

internal/config/backoff.go

@@ -28,6 +28,7 @@ type Backoff struct {
 	BackoffFactor    float64 `json:"backoff_factor"     yaml:"backoff_factor"`
 	RetryCount       int     `json:"retry_count"        yaml:"retry_count"`
 	EnableErrorLog   bool    `json:"enable_error_log"   yaml:"enable_error_log"`
+	EnableMetrics    bool    `json:"enable_metrics"     yaml:"enable_metrics"`
 }
 
 // Bind binds the actual data from the Backoff receiver fields.
@@ -57,5 +58,21 @@ func (b *Backoff) Opts() []backoff.Option {
 		)
 	}
 
+	if b.EnableMetrics {
+		opts = append(opts,
+			backoff.WithEnableMetrics(),
+		)
+	}
+
 	return opts
 }
+
+func (b *Backoff) CreateBackoff() backoff.Backoff {
+	if len(b.InitialDuration) != 0 &&
+		b.RetryCount > 2 {
+		return backoff.New(
+			b.Opts()...,
+		)
+	}
+	return nil
+}

internal/net/grpc/client.go

@@ -550,6 +550,9 @@ func (g *gRPCClient) do(ctx context.Context, p pool.Conn, addr string, enableBac
 		}
 	}()
 	if g.bo != nil && enableBackoff {
+		if method := FromGRPCMethod(sctx); len(method) != 0 {
+			sctx = backoff.WithBackoffName(ctx, method+"/"+addr)
+		}
 		data, err = g.bo.Do(sctx, func(ictx context.Context) (r interface{}, ret bool, err error) {
 			err = p.Do(func(conn *ClientConn) (err error) {
 				if conn == nil {

internal/net/grpc/context.go

@@ -0,0 +1,35 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package grpc
+
+import "context"
+
+type contextKey string
+
+const grpcMethodContextKey contextKey = "grpc_method"
+
+// WithGRPCMethod returns a copy of parent in which the method associated with key (grpcMethodContextKey).
+func WithGRPCMethod(ctx context.Context, method string) context.Context {
+	return context.WithValue(ctx, grpcMethodContextKey, method)
+}
+
+// FromGRPCMethod returns the value associated with this context for key (grpcMethodContextKey).
+func FromGRPCMethod(ctx context.Context) string {
+	if v := ctx.Value(grpcMethodContextKey); v != nil {
+		return v.(string)
+	}
+	return ""
+}

internal/net/http/client/client_test.go

@@ -49,7 +49,7 @@ var (
 		comparator.AllowUnexported(http.Transport{}),
 		comparator.IgnoreFields(http.Transport{}, "idleLRU", "altProto", "TLSNextProto"),
 		comparator.Exporter(func(t reflect.Type) bool {
-			if t.Name() == "ert" || t.Name() == "backoff" {
+			if t.Name() == "ert" || t.Name() == "backoff" || t.Name() == "RWMutex" {
 				return true
 			}
 			return false

internal/net/http/transport/roundtrip_mock.go

@@ -29,14 +29,19 @@ func (rm *roundTripMock) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 type backoffMock struct {
-	DoFunc    func(context.Context, func(context.Context) (interface{}, bool, error)) (interface{}, error)
-	CloseFunc func()
+	DoFunc      func(context.Context, func(context.Context) (interface{}, bool, error)) (interface{}, error)
+	MetricsFunc func(ctx context.Context) map[string]int64
+	CloseFunc   func()
 }
 
 func (bm *backoffMock) Do(ctx context.Context, fn func(context.Context) (interface{}, bool, error)) (interface{}, error) {
 	return bm.DoFunc(ctx, fn)
 }
 
+func (bm *backoffMock) Metrics(ctx context.Context) map[string]int64 {
+	return bm.MetricsFunc(ctx)
+}
+
 func (bm *backoffMock) Close() {
 	bm.CloseFunc()
 }