Add backoff metrics for grpc

apis/grpc/v1/vald/vald.go

@@ -63,6 +63,68 @@ type ClientWithFilter interface {
 	FilterClient
 }
 
+const PackageName = "vald.v1"
+
+const (
+	InsertRPCServiceName = "Insert"
+	UpdateRPCServiceName = "Update"
+	UpsertRPCServiceName = "Upsert"
+	SearchRPCServiceName = "Search"
+	RemoveRPCServiceName = "Remove"
+	ObjectRPCServiceName = "Object"
+	FilterRPCServiceName = "Filter"
+)
+
+const (
+	InsertRPCName             = "Insert"
+	StreamInsertRPCName       = "StreamInsert"
+	MultiInsertRPCName        = "MultiInsert"
+	InsertObjectRPCName       = "InsertObject"
+	StreamInsertObjectRPCName = "StreamInsertObject"
+	MultiInsertObjectRPCName  = "MultiInsertObject"
+
+	UpdateRPCName             = "Update"
+	StreamUpdateRPCName       = "StreamUpdate"
+	MultiUpdateRPCName        = "MultiUpdate"
+	UpdateObjectRPCName       = "UpdateObject"
+	StreamUpdateObjectRPCName = "StreamUpdateObject"
+	MultiUpdateObjectRPCName  = "MultiUpdateObject"
+
+	UpsertRPCName             = "Upsert"
+	StreamUpsertRPCName       = "StreamUpsert"
+	MultiUpsertRPCName        = "MultiUpsert"
+	UpsertObjectRPCName       = "UpsertObject"
+	StreamUpsertObjectRPCName = "StreamUpsertObject"
+	MultiUpsertObjectRPCName  = "MultiUpsertObject"
+
+	SearchRPCName                   = "Search"
+	SearchByIDRPCName               = "SearchByID"
+	StreamSearchRPCName             = "StreamSearch"
+	StreamSearchByIDRPCName         = "StreamSearchByID"
+	MultiSearchRPCName              = "MultiSearch"
+	MultiSearchByIDRPCName          = "MultiSearchByID"
+	LinearSearchRPCName             = "LinearSearch"
+	LinearSearchByIDRPCName         = "LinearSearchByID"
+	StreamLinearSearchRPCName       = "StreamLinearSearch"
+	StreamLinearSearchByIDRPCName   = "StreamLinearSearchByID"
+	MultiLinearSearchRPCName        = "MultiLinearSearch"
+	MultiLinearSearchByIDRPCName    = "MultiLinearSearchByID"
+	SearchObjectRPCName             = "SearchObject"
+	MultiSearchObjectRPCName        = "MultiSearchObject"
+	LinearSearchObjectRPCName       = "LinearSearchObject"
+	MultiLinearSearchObjectRPCName  = "MultiLinearSearchObject"
+	StreamLinearSearchObjectRPCName = "StreamLinearSearchObject"
+	StreamSearchObjectRPCName       = "StreamSearchObject"
+
+	RemoveRPCName       = "Remove"
+	StreamRemoveRPCName = "StreamRemove"
+	MultiRemoveRPCName  = "MultiRemove"
+
+	ExistsRPCName          = "Exists"
+	GetObjectRPCName       = "GetObject"
+	StreamGetObjectRPCName = "StreamGetObject"
+)
+
 type client struct {
 	InsertClient
 	UpdateClient

internal/backoff/backoff.go

@@ -30,6 +30,13 @@ import (
 	"github.com/vdaas/vald/internal/rand"
 )
 
+// NOTE: This variable is for observability package.
+//       This will be fixed when refactoring the observability package.
+var (
+	mu      sync.RWMutex
+	metrics map[string]int64 = make(map[string]int64)
+)
+
 type backoff struct {
 	wg                    sync.WaitGroup
 	backoffFactor         float64
@@ -86,6 +93,7 @@ func (b *backoff) Do(ctx context.Context, f func(ctx context.Context) (val inter
 
 	dur := b.initialDuration
 	jdur := b.jittedInitialDuration
+	name := FromBackoffName(ctx)
 
 	dctx, cancel := context.WithDeadline(sctx, time.Now().Add(b.backoffTimeLimit))
 	defer cancel()
@@ -119,6 +127,13 @@ func (b *backoff) Do(ctx context.Context, f func(ctx context.Context) (val inter
 			if b.errLog {
 				log.Error(err)
 			}
+			// e.g. name = vald.v1.Exists/ip ...etc
+			if len(name) != 0 {
+				mu.Lock()
+				metrics[name] += 1
+				mu.Unlock()
+			}
+
 			timer.Reset(time.Duration(jdur))
 			select {
 			case <-dctx.Done():
@@ -153,3 +168,18 @@ func (b *backoff) addJitter(dur float64) float64 {
 func (b *backoff) Close() {
 	b.wg.Wait()
 }
+
+func Metrics(_ context.Context) map[string]int64 {
+	mu.RLock()
+	defer mu.RUnlock()
+
+	if len(metrics) == 0 {
+		return nil
+	}
+
+	m := make(map[string]int64, len(metrics))
+	for name, cnt := range metrics {
+		m[name] = cnt
+	}
+	return m
+}

internal/backoff/backoff_test.go

@@ -617,3 +617,77 @@ func Test_backoff_Do(t *testing.T) {
 		})
 	}
 }
+
+func TestMetrics(t *testing.T) {
+	type args struct {
+		in0 context.Context
+	}
+	type want struct {
+		want map[string]int64
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, map[string]int64) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got map[string]int64) error {
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+		}
+		return nil
+	}
+	tests := []test{
+		// TODO test cases
+		/*
+		   {
+		       name: "test_case_1",
+		       args: args {
+		           in0: nil,
+		       },
+		       want: want{},
+		       checkFunc: defaultCheckFunc,
+		   },
+		*/
+
+		// TODO test cases
+		/*
+		   func() test {
+		       return test {
+		           name: "test_case_2",
+		           args: args {
+		           in0: nil,
+		           },
+		           want: want{},
+		           checkFunc: defaultCheckFunc,
+		       }
+		   }(),
+		*/
+	}
+
+	for _, tc := range tests {
+		test := tc
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			checkFunc := test.checkFunc
+			if test.checkFunc == nil {
+				checkFunc = defaultCheckFunc
+			}
+
+			got := Metrics(test.args.in0)
+			if err := checkFunc(test.want, got); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+
+		})
+	}
+}

internal/backoff/context.go

@@ -0,0 +1,37 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package backoff
+
+import "context"
+
+type contextKey string
+
+const backoffNameContextKey contextKey = "backoff_name"
+
+// WithBackoffName returns a copy of parent in which the method associated with key (backoffNameContextKey).
+func WithBackoffName(ctx context.Context, name string) context.Context {
+	return context.WithValue(ctx, backoffNameContextKey, name)
+}
+
+// FromBackoffName returns the value associated with this context for key (backoffNameContextKey).
+func FromBackoffName(ctx context.Context) string {
+	if val := ctx.Value(backoffNameContextKey); val != nil {
+		if name, ok := val.(string); ok {
+			return name
+		}
+	}
+	return ""
+}