diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index fa95bdcaa7d4..62ad6e2d2a0b 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -127,6 +127,12 @@ var ( Measurement: "Lease Requests", Unit: metric.Unit_COUNT, } + metaLeaseRequestLatency = metric.Metadata{ + Name: "leases.requests.latency", + Help: "Lease request latency (all types and outcomes, coalesced)", + Measurement: "Latency", + Unit: metric.Unit_NANOSECONDS, + } metaLeaseTransferSuccessCount = metric.Metadata{ Name: "leases.transfers.success", Help: "Number of successful lease transfers", @@ -1797,6 +1803,7 @@ type StoreMetrics struct { // lease). LeaseRequestSuccessCount *metric.Counter LeaseRequestErrorCount *metric.Counter + LeaseRequestLatency metric.IHistogram LeaseTransferSuccessCount *metric.Counter LeaseTransferErrorCount *metric.Counter LeaseExpirationCount *metric.Gauge @@ -2355,8 +2362,14 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { OverReplicatedRangeCount: metric.NewGauge(metaOverReplicatedRangeCount), // Lease request metrics. - LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount), - LeaseRequestErrorCount: metric.NewCounter(metaLeaseRequestErrorCount), + LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount), + LeaseRequestErrorCount: metric.NewCounter(metaLeaseRequestErrorCount), + LeaseRequestLatency: metric.NewHistogram(metric.HistogramOptions{ + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaLeaseRequestLatency, + Duration: histogramWindow, + Buckets: metric.NetworkLatencyBuckets, + }), LeaseTransferSuccessCount: metric.NewCounter(metaLeaseTransferSuccessCount), LeaseTransferErrorCount: metric.NewCounter(metaLeaseTransferErrorCount), LeaseExpirationCount: metric.NewGauge(metaLeaseExpirationCount), diff --git a/pkg/kv/kvserver/replica_range_lease.go b/pkg/kv/kvserver/replica_range_lease.go index c81cc8c23651..d8ce4dc077b3 100644 --- a/pkg/kv/kvserver/replica_range_lease.go +++ b/pkg/kv/kvserver/replica_range_lease.go @@ -429,6 +429,11 @@ func (p *pendingLeaseRequest) requestLease( status kvserverpb.LeaseStatus, leaseReq kvpb.Request, ) error { + started := timeutil.Now() + defer func() { + p.repl.store.metrics.LeaseRequestLatency.RecordValue(timeutil.Since(started).Nanoseconds()) + }() + // If requesting an epoch-based lease & current state is expired, // potentially heartbeat our own liveness or increment epoch of // prior owner. Note we only do this if the previous lease was diff --git a/pkg/ts/catalog/chart_catalog.go b/pkg/ts/catalog/chart_catalog.go index 684a93288cbf..72494d1d92ca 100644 --- a/pkg/ts/catalog/chart_catalog.go +++ b/pkg/ts/catalog/chart_catalog.go @@ -1757,6 +1757,10 @@ var charts = []sectionDescription{ Title: "Stuck Acquisition Count", Metrics: []string{"requests.slow.lease"}, }, + { + Title: "Lease Request Latency", + Metrics: []string{"leases.requests.latency"}, + }, { Title: "Succcess Rate", Metrics: []string{ diff --git a/pkg/ts/catalog/metrics.go b/pkg/ts/catalog/metrics.go index 29fdb2ea422f..4b6c35167528 100644 --- a/pkg/ts/catalog/metrics.go +++ b/pkg/ts/catalog/metrics.go @@ -103,6 +103,7 @@ var histogramMetricsNames = map[string]struct{}{ "replication.flush_hist_nanos": {}, "kv.replica_read_batch_evaluate.latency": {}, "kv.replica_write_batch_evaluate.latency": {}, + "leases.requests.latency": {}, } func allInternalTSMetricsNames() []string {