From 5dab5e7b1e7fd26be3e56f895598067710953f93 Mon Sep 17 00:00:00 2001
From: pingcap-github-bot <sre-bot@pingcap.com>
Date: Fri, 16 Aug 2019 12:49:34 +0800
Subject: [PATCH] metric, tikv: record duration for each backoff type (#11710)
 (#11728)

---
 metrics/tikvclient.go     |  4 ++--
 store/tikv/backoff.go     | 49 ++++++++++++++++++++++++---------------
 store/tikv/coprocessor.go |  3 ---
 3 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/metrics/tikvclient.go b/metrics/tikvclient.go
index c52c8a33e0dde..f4cb704f7d38a 100644
--- a/metrics/tikvclient.go
+++ b/metrics/tikvclient.go
@@ -58,14 +58,14 @@ var (
 			Help:      "Counter of backoff.",
 		}, []string{LblType})
 
-	TiKVBackoffHistogram = prometheus.NewHistogram(
+	TiKVBackoffHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: "tidb",
 			Subsystem: "tikvclient",
 			Name:      "backoff_seconds",
 			Help:      "total backoff seconds of a single backoffer.",
 			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 524s
-		})
+		}, []string{LblType})
 
 	TiKVSendReqHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go
index 4f6ae1be22515..5319e66a770b9 100644
--- a/store/tikv/backoff.go
+++ b/store/tikv/backoff.go
@@ -45,34 +45,42 @@ const (
 )
 
 var (
-	tikvBackoffCounterRPC          = metrics.TiKVBackoffCounter.WithLabelValues("tikvRPC")
-	tikvBackoffCounterLock         = metrics.TiKVBackoffCounter.WithLabelValues("txnLock")
-	tikvBackoffCounterLockFast     = metrics.TiKVBackoffCounter.WithLabelValues("tikvLockFast")
-	tikvBackoffCounterPD           = metrics.TiKVBackoffCounter.WithLabelValues("pdRPC")
-	tikvBackoffCounterRegionMiss   = metrics.TiKVBackoffCounter.WithLabelValues("regionMiss")
-	tikvBackoffCounterUpdateLeader = metrics.TiKVBackoffCounter.WithLabelValues("updateLeader")
-	tikvBackoffCounterServerBusy   = metrics.TiKVBackoffCounter.WithLabelValues("serverBusy")
-	tikvBackoffCounterEmpty        = metrics.TiKVBackoffCounter.WithLabelValues("")
+	tikvBackoffCounterRPC            = metrics.TiKVBackoffCounter.WithLabelValues("tikvRPC")
+	tikvBackoffCounterLock           = metrics.TiKVBackoffCounter.WithLabelValues("txnLock")
+	tikvBackoffCounterLockFast       = metrics.TiKVBackoffCounter.WithLabelValues("tikvLockFast")
+	tikvBackoffCounterPD             = metrics.TiKVBackoffCounter.WithLabelValues("pdRPC")
+	tikvBackoffCounterRegionMiss     = metrics.TiKVBackoffCounter.WithLabelValues("regionMiss")
+	tikvBackoffCounterUpdateLeader   = metrics.TiKVBackoffCounter.WithLabelValues("updateLeader")
+	tikvBackoffCounterServerBusy     = metrics.TiKVBackoffCounter.WithLabelValues("serverBusy")
+	tikvBackoffCounterEmpty          = metrics.TiKVBackoffCounter.WithLabelValues("")
+	tikvBackoffHistogramRPC          = metrics.TiKVBackoffHistogram.WithLabelValues("tikvRPC")
+	tikvBackoffHistogramLock         = metrics.TiKVBackoffHistogram.WithLabelValues("txnLock")
+	tikvBackoffHistogramLockFast     = metrics.TiKVBackoffHistogram.WithLabelValues("tikvLockFast")
+	tikvBackoffHistogramPD           = metrics.TiKVBackoffHistogram.WithLabelValues("pdRPC")
+	tikvBackoffHistogramRegionMiss   = metrics.TiKVBackoffHistogram.WithLabelValues("regionMiss")
+	tikvBackoffHistogramUpdateLeader = metrics.TiKVBackoffHistogram.WithLabelValues("updateLeader")
+	tikvBackoffHistogramServerBusy   = metrics.TiKVBackoffHistogram.WithLabelValues("serverBusy")
+	tikvBackoffHistogramEmpty        = metrics.TiKVBackoffHistogram.WithLabelValues("")
 )
 
-func (t backoffType) Counter() prometheus.Counter {
+func (t backoffType) metric() (prometheus.Counter, prometheus.Observer) {
 	switch t {
 	case boTiKVRPC:
-		return tikvBackoffCounterRPC
+		return tikvBackoffCounterRPC, tikvBackoffHistogramRPC
 	case BoTxnLock:
-		return tikvBackoffCounterLock
+		return tikvBackoffCounterLock, tikvBackoffHistogramLock
 	case boTxnLockFast:
-		return tikvBackoffCounterLockFast
+		return tikvBackoffCounterLockFast, tikvBackoffHistogramLockFast
 	case BoPDRPC:
-		return tikvBackoffCounterPD
+		return tikvBackoffCounterPD, tikvBackoffHistogramPD
 	case BoRegionMiss:
-		return tikvBackoffCounterRegionMiss
+		return tikvBackoffCounterRegionMiss, tikvBackoffHistogramRegionMiss
 	case BoUpdateLeader:
-		return tikvBackoffCounterUpdateLeader
+		return tikvBackoffCounterUpdateLeader, tikvBackoffHistogramUpdateLeader
 	case boServerBusy:
-		return tikvBackoffCounterServerBusy
+		return tikvBackoffCounterServerBusy, tikvBackoffHistogramServerBusy
 	}
-	return tikvBackoffCounterEmpty
+	return tikvBackoffCounterEmpty, tikvBackoffHistogramEmpty
 }
 
 // NewBackoffFn creates a backoff func which implements exponential backoff with
@@ -276,7 +284,8 @@ func (b *Backoffer) BackoffWithMaxSleep(typ backoffType, maxSleepMs int, err err
 	default:
 	}
 
-	typ.Counter().Inc()
+	backoffCounter, backoffDuration := typ.metric()
+	backoffCounter.Inc()
 	// Lazy initialize.
 	if b.fn == nil {
 		b.fn = make(map[backoffType]func(context.Context, int) int)
@@ -287,7 +296,9 @@ func (b *Backoffer) BackoffWithMaxSleep(typ backoffType, maxSleepMs int, err err
 		b.fn[typ] = f
 	}
 
-	b.totalSleep += f(b.ctx, maxSleepMs)
+	realSleep := f(b.ctx, maxSleepMs)
+	backoffDuration.Observe(float64(realSleep) / 1000)
+	b.totalSleep += realSleep
 	b.types = append(b.types, typ)
 
 	var startTs interface{}
diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go
index 168cf1b6ce3d3..712de3f1b7b30 100644
--- a/store/tikv/coprocessor.go
+++ b/store/tikv/coprocessor.go
@@ -473,9 +473,6 @@ func (worker *copIteratorWorker) run(ctx context.Context) {
 
 		bo := NewBackoffer(ctx, copNextMaxBackoff).WithVars(worker.vars)
 		worker.handleTask(bo, task, respCh)
-		if bo.totalSleep > 0 {
-			metrics.TiKVBackoffHistogram.Observe(float64(bo.totalSleep) / 1000)
-		}
 		close(task.respChan)
 		select {
 		case <-worker.finishCh: