Merge pull request #100474 from cockroachdb/blathers/backport-release…

…-22.2-100287 release-22.2: kvserver: add `leases.requests.latency` metric
cockroachdb · Apr 30, 2023 · bb0e721 · bb0e721
2 parents f013742 + 8e7edf0
commit bb0e721
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 2 deletions.
diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go
@@ -124,6 +124,12 @@ var (
 		Measurement: "Lease Requests",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaLeaseRequestLatency = metric.Metadata{
+		Name:        "leases.requests.latency",
+		Help:        "Lease request latency (all types and outcomes, coalesced)",
+		Measurement: "Latency",
+		Unit:        metric.Unit_NANOSECONDS,
+	}
 	metaLeaseTransferSuccessCount = metric.Metadata{
 		Name:        "leases.transfers.success",
 		Help:        "Number of successful lease transfers",
@@ -1663,6 +1669,7 @@ type StoreMetrics struct {
 	// lease).
 	LeaseRequestSuccessCount  *metric.Counter
 	LeaseRequestErrorCount    *metric.Counter
+	LeaseRequestLatency       metric.IHistogram
 	LeaseTransferSuccessCount *metric.Counter
 	LeaseTransferErrorCount   *metric.Counter
 	LeaseExpirationCount      *metric.Gauge
@@ -2189,8 +2196,14 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		OverReplicatedRangeCount:  metric.NewGauge(metaOverReplicatedRangeCount),
 
 		// Lease request metrics.
-		LeaseRequestSuccessCount:  metric.NewCounter(metaLeaseRequestSuccessCount),
-		LeaseRequestErrorCount:    metric.NewCounter(metaLeaseRequestErrorCount),
+		LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount),
+		LeaseRequestErrorCount:   metric.NewCounter(metaLeaseRequestErrorCount),
+		LeaseRequestLatency: metric.NewHistogram(metric.HistogramOptions{
+			Mode:     metric.HistogramModePreferHdrLatency,
+			Metadata: metaLeaseRequestLatency,
+			Duration: histogramWindow,
+			Buckets:  metric.NetworkLatencyBuckets,
+		}),
 		LeaseTransferSuccessCount: metric.NewCounter(metaLeaseTransferSuccessCount),
 		LeaseTransferErrorCount:   metric.NewCounter(metaLeaseTransferErrorCount),
 		LeaseExpirationCount:      metric.NewGauge(metaLeaseExpirationCount),

diff --git a/pkg/kv/kvserver/replica_range_lease.go b/pkg/kv/kvserver/replica_range_lease.go
@@ -449,6 +449,11 @@ func (p *pendingLeaseRequest) requestLease(
 	status kvserverpb.LeaseStatus,
 	leaseReq roachpb.Request,
 ) error {
+	started := timeutil.Now()
+	defer func() {
+		p.repl.store.metrics.LeaseRequestLatency.RecordValue(timeutil.Since(started).Nanoseconds())
+	}()
+
 	// If requesting an epoch-based lease & current state is expired,
 	// potentially heartbeat our own liveness or increment epoch of
 	// prior owner. Note we only do this if the previous lease was

diff --git a/pkg/ts/catalog/chart_catalog.go b/pkg/ts/catalog/chart_catalog.go
@@ -1670,6 +1670,10 @@ var charts = []sectionDescription{
 				Title:   "Stuck Acquisition Count",
 				Metrics: []string{"requests.slow.lease"},
 			},
+			{
+				Title:   "Lease Request Latency",
+				Metrics: []string{"leases.requests.latency"},
+			},
 			{
 				Title: "Succcess Rate",
 				Metrics: []string{

diff --git a/pkg/ts/catalog/metrics.go b/pkg/ts/catalog/metrics.go
@@ -102,6 +102,7 @@ var histogramMetricsNames = map[string]struct{}{
 	"streaming.flush_hist_nanos":                {},
 	"kv.replica_read_batch_evaluate.latency":    {},
 	"kv.replica_write_batch_evaluate.latency":   {},
+	"leases.requests.latency":                   {},
 }
 
 func allInternalTSMetricsNames() []string {