From 49a76ade29baa469ddcf8fceb39b507d7c81a3ae Mon Sep 17 00:00:00 2001
From: Paul Bardea <pbardea@gmail.com>
Date: Wed, 7 Jul 2021 10:15:39 -0400
Subject: [PATCH] backupccl: add exportrequest.delay.total metric

This commit adds a metric to track how long export requests are blocked
by the concurrency limiter.

Release note (ops change): Introduce a new metric
(exportrequest.delay.total) to track how long ExportRequests (issued by
BACKUP) are delayed by throttling mechansisms.
---
 pkg/kv/kvserver/metrics.go      | 14 ++++++++++++++
 pkg/kv/kvserver/store_send.go   |  1 +
 pkg/ts/catalog/chart_catalog.go | 11 +++++++++++
 3 files changed, 26 insertions(+)

diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go
index 02fe7da0a40b..b31d2c1ccbff 100644
--- a/pkg/kv/kvserver/metrics.go
+++ b/pkg/kv/kvserver/metrics.go
@@ -1051,6 +1051,14 @@ var (
 		Unit:        metric.Unit_NANOSECONDS,
 	}
 
+	// Export request counter.
+	metaExportEvalTotalDelay = metric.Metadata{
+		Name:        "exportrequest.delay.total",
+		Help:        "Amount by which evaluation of Export requests was delayed",
+		Measurement: "Nanoseconds",
+		Unit:        metric.Unit_NANOSECONDS,
+	}
+
 	// Encryption-at-rest metrics.
 	// TODO(mberhault): metrics for key age, per-key file/bytes counts.
 	metaEncryptionAlgorithm = metric.Metadata{
@@ -1282,6 +1290,9 @@ type StoreMetrics struct {
 	AddSSTableProposalTotalDelay  *metric.Counter
 	AddSSTableProposalEngineDelay *metric.Counter
 
+	// Export request stats.
+	ExportRequestProposalTotalDelay *metric.Counter
+
 	// Encryption-at-rest stats.
 	// EncryptionAlgorithm is an enum representing the cipher in use, so we use a gauge.
 	EncryptionAlgorithm *metric.Gauge
@@ -1672,6 +1683,9 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		AddSSTableProposalTotalDelay:  metric.NewCounter(metaAddSSTableEvalTotalDelay),
 		AddSSTableProposalEngineDelay: metric.NewCounter(metaAddSSTableEvalEngineDelay),
 
+		// ExportRequest proposal.
+		ExportRequestProposalTotalDelay: metric.NewCounter(metaExportEvalTotalDelay),
+
 		// Encryption-at-rest.
 		EncryptionAlgorithm: metric.NewGauge(metaEncryptionAlgorithm),
 
diff --git a/pkg/kv/kvserver/store_send.go b/pkg/kv/kvserver/store_send.go
index 8a6cd885de58..cdb21b7349e4 100644
--- a/pkg/kv/kvserver/store_send.go
+++ b/pkg/kv/kvserver/store_send.go
@@ -299,6 +299,7 @@ func (s *Store) maybeThrottleBatch(
 		}
 
 		waited := timeutil.Since(before)
+		s.metrics.ExportRequestProposalTotalDelay.Inc(waited.Nanoseconds())
 		if waited > time.Second {
 			log.Infof(ctx, "Export request was delayed by %v", waited)
 		}
diff --git a/pkg/ts/catalog/chart_catalog.go b/pkg/ts/catalog/chart_catalog.go
index 55f90aa746d5..975067f9e313 100644
--- a/pkg/ts/catalog/chart_catalog.go
+++ b/pkg/ts/catalog/chart_catalog.go
@@ -2305,6 +2305,17 @@ var charts = []sectionDescription{
 			},
 		},
 	},
+	{
+		Organization: [][]string{{DistributionLayer, "Bulk", "Egress"}},
+		Charts: []chartDescription{
+			{
+				Title: "Export Delays",
+				Metrics: []string{
+					"exportrequest.delay.total",
+				},
+			},
+		},
+	},
 	{
 		Organization: [][]string{{StorageLayer, "Storage", "KV"}},
 		Charts: []chartDescription{