Merge pull request #109805 from jmcarp/backport23.1-107388

release-23.1: metrics: refactor histogram bucket generation and testing

pkg/ccl/changefeedccl/metrics.go

@@ -632,52 +632,52 @@ func newAggregateMetrics(histogramWindow time.Duration) *AggMetrics {
 		EmittedMessages:  b.Counter(metaChangefeedEmittedMessages),
 		FilteredMessages: b.Counter(metaChangefeedFilteredMessages),
 		MessageSize: b.Histogram(metric.HistogramOptions{
-			Metadata: metaMessageSize,
-			Duration: histogramWindow,
-			MaxVal:   10 << 20, /* 10MB max message size */
-			SigFigs:  1,
-			Buckets:  metric.DataSize16MBBuckets,
+			Metadata:     metaMessageSize,
+			Duration:     histogramWindow,
+			MaxVal:       10 << 20, /* 10MB max message size */
+			SigFigs:      1,
+			BucketConfig: metric.DataSize16MBBuckets,
 		}),
 		EmittedBytes:     b.Counter(metaChangefeedEmittedBytes),
 		FlushedBytes:     b.Counter(metaChangefeedFlushedBytes),
 		Flushes:          b.Counter(metaChangefeedFlushes),
 		SizeBasedFlushes: b.Counter(metaSizeBasedFlushes),
 		ParallelIOQueueNanos: b.Histogram(metric.HistogramOptions{
-			Metadata: metaChangefeedParallelIOQueueNanos,
-			Duration: histogramWindow,
-			MaxVal:   changefeedIOQueueMaxLatency.Nanoseconds(),
-			SigFigs:  2,
-			Buckets:  metric.BatchProcessLatencyBuckets,
+			Metadata:     metaChangefeedParallelIOQueueNanos,
+			Duration:     histogramWindow,
+			MaxVal:       changefeedIOQueueMaxLatency.Nanoseconds(),
+			SigFigs:      2,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
 		}),
 		SinkIOInflight: b.Gauge(metaChangefeedSinkIOInflight),
 
 		BatchHistNanos: b.Histogram(metric.HistogramOptions{
-			Metadata: metaChangefeedBatchHistNanos,
-			Duration: histogramWindow,
-			MaxVal:   changefeedBatchHistMaxLatency.Nanoseconds(),
-			SigFigs:  1,
-			Buckets:  metric.BatchProcessLatencyBuckets,
+			Metadata:     metaChangefeedBatchHistNanos,
+			Duration:     histogramWindow,
+			MaxVal:       changefeedBatchHistMaxLatency.Nanoseconds(),
+			SigFigs:      1,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
 		}),
 		FlushHistNanos: b.Histogram(metric.HistogramOptions{
-			Metadata: metaChangefeedFlushHistNanos,
-			Duration: histogramWindow,
-			MaxVal:   changefeedFlushHistMaxLatency.Nanoseconds(),
-			SigFigs:  2,
-			Buckets:  metric.BatchProcessLatencyBuckets,
+			Metadata:     metaChangefeedFlushHistNanos,
+			Duration:     histogramWindow,
+			MaxVal:       changefeedFlushHistMaxLatency.Nanoseconds(),
+			SigFigs:      2,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
 		}),
 		CommitLatency: b.Histogram(metric.HistogramOptions{
-			Metadata: metaCommitLatency,
-			Duration: histogramWindow,
-			MaxVal:   commitLatencyMaxValue.Nanoseconds(),
-			SigFigs:  1,
-			Buckets:  metric.BatchProcessLatencyBuckets,
+			Metadata:     metaCommitLatency,
+			Duration:     histogramWindow,
+			MaxVal:       commitLatencyMaxValue.Nanoseconds(),
+			SigFigs:      1,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
 		}),
 		AdmitLatency: b.Histogram(metric.HistogramOptions{
-			Metadata: metaAdmitLatency,
-			Duration: histogramWindow,
-			MaxVal:   admitLatencyMaxValue.Nanoseconds(),
-			SigFigs:  1,
-			Buckets:  metric.BatchProcessLatencyBuckets,
+			Metadata:     metaAdmitLatency,
+			Duration:     histogramWindow,
+			MaxVal:       admitLatencyMaxValue.Nanoseconds(),
+			SigFigs:      1,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
 		}),
 		BackfillCount:             b.Gauge(metaChangefeedBackfillCount),
 		BackfillPendingRanges:     b.Gauge(metaChangefeedBackfillPendingRanges),
@@ -820,28 +820,28 @@ func MakeMetrics(histogramWindow time.Duration) metric.Struct {
 		Failures:          metric.NewCounter(metaChangefeedFailures),
 		QueueTimeNanos:    metric.NewCounter(metaEventQueueTime),
 		CheckpointHistNanos: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaChangefeedCheckpointHistNanos,
-			Duration: histogramWindow,
-			MaxVal:   changefeedCheckpointHistMaxLatency.Nanoseconds(),
-			SigFigs:  2,
-			Buckets:  metric.IOLatencyBuckets,
+			Metadata:     metaChangefeedCheckpointHistNanos,
+			Duration:     histogramWindow,
+			MaxVal:       changefeedCheckpointHistMaxLatency.Nanoseconds(),
+			SigFigs:      2,
+			BucketConfig: metric.IOLatencyBuckets,
 		}),
 		FrontierUpdates: metric.NewCounter(metaChangefeedFrontierUpdates),
 		ThrottleMetrics: cdcutils.MakeMetrics(histogramWindow),
 		ReplanCount:     metric.NewCounter(metaChangefeedReplanCount),
 		// Below two metrics were never implemented using the hdr histogram. Set ForceUsePrometheus
 		// to true.
 		ParallelConsumerFlushNanos: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaChangefeedEventConsumerFlushNanos,
-			Duration: histogramWindow,
-			Buckets:  metric.IOLatencyBuckets,
-			Mode:     metric.HistogramModePrometheus,
+			Metadata:     metaChangefeedEventConsumerFlushNanos,
+			Duration:     histogramWindow,
+			BucketConfig: metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePrometheus,
 		}),
 		ParallelConsumerConsumeNanos: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaChangefeedEventConsumerConsumeNanos,
-			Duration: histogramWindow,
-			Buckets:  metric.IOLatencyBuckets,
-			Mode:     metric.HistogramModePrometheus,
+			Metadata:     metaChangefeedEventConsumerConsumeNanos,
+			Duration:     histogramWindow,
+			BucketConfig: metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePrometheus,
 		}),
 		ParallelConsumerInFlightEvents: metric.NewGauge(metaChangefeedEventConsumerInFlightEvents),
 	}

pkg/ccl/sqlproxyccl/connector_test.go

@@ -381,10 +381,10 @@ func TestConnector_dialTenantCluster(t *testing.T) {
 		c := &connector{
 			TenantID: roachpb.MustMakeTenantID(42),
 			DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePrometheus,
-				Metadata: metaDialTenantLatency,
-				Duration: time.Millisecond,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePrometheus,
+				Metadata:     metaDialTenantLatency,
+				Duration:     time.Millisecond,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 			DialTenantRetries: metric.NewCounter(metaDialTenantRetries),
 		}
@@ -466,10 +466,10 @@ func TestConnector_dialTenantCluster(t *testing.T) {
 
 		c := &connector{
 			DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePreferHdrLatency,
-				Metadata: metaDialTenantLatency,
-				Duration: time.Millisecond,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePreferHdrLatency,
+				Metadata:     metaDialTenantLatency,
+				Duration:     time.Millisecond,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 			DialTenantRetries: metric.NewCounter(metaDialTenantRetries),
 		}
@@ -500,10 +500,10 @@ func TestConnector_dialTenantCluster(t *testing.T) {
 		c := &connector{
 			TenantID: roachpb.MustMakeTenantID(42),
 			DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePreferHdrLatency,
-				Metadata: metaDialTenantLatency,
-				Duration: time.Millisecond,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePreferHdrLatency,
+				Metadata:     metaDialTenantLatency,
+				Duration:     time.Millisecond,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 			DialTenantRetries: metric.NewCounter(metaDialTenantRetries),
 		}

pkg/ccl/sqlproxyccl/metrics.go

@@ -234,19 +234,19 @@ func makeProxyMetrics() metrics {
 		RefusedConnCount:       metric.NewCounter(metaRefusedConnCount),
 		SuccessfulConnCount:    metric.NewCounter(metaSuccessfulConnCount),
 		ConnectionLatency: metric.NewHistogram(metric.HistogramOptions{
-			Mode:     metric.HistogramModePreferHdrLatency,
-			Metadata: metaConnMigrationAttemptedCount,
-			Duration: base.DefaultHistogramWindowInterval(),
-			Buckets:  metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePreferHdrLatency,
+			Metadata:     metaConnMigrationAttemptedCount,
+			Duration:     base.DefaultHistogramWindowInterval(),
+			BucketConfig: metric.IOLatencyBuckets,
 		}),
 		AuthFailedCount:        metric.NewCounter(metaAuthFailedCount),
 		ExpiredClientConnCount: metric.NewCounter(metaExpiredClientConnCount),
 		// Connector metrics.
 		DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{
-			Mode:     metric.HistogramModePreferHdrLatency,
-			Metadata: metaDialTenantLatency,
-			Duration: base.DefaultHistogramWindowInterval(),
-			Buckets:  metric.IOLatencyBuckets},
+			Mode:         metric.HistogramModePreferHdrLatency,
+			Metadata:     metaDialTenantLatency,
+			Duration:     base.DefaultHistogramWindowInterval(),
+			BucketConfig: metric.IOLatencyBuckets},
 		),
 		DialTenantRetries: metric.NewCounter(metaDialTenantRetries),
 		// Connection migration metrics.
@@ -255,17 +255,17 @@ func makeProxyMetrics() metrics {
 		ConnMigrationErrorRecoverableCount: metric.NewCounter(metaConnMigrationErrorRecoverableCount),
 		ConnMigrationAttemptedCount:        metric.NewCounter(metaConnMigrationAttemptedCount),
 		ConnMigrationAttemptedLatency: metric.NewHistogram(metric.HistogramOptions{
-			Mode:     metric.HistogramModePreferHdrLatency,
-			Metadata: metaConnMigrationAttemptedLatency,
-			Duration: base.DefaultHistogramWindowInterval(),
-			Buckets:  metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePreferHdrLatency,
+			Metadata:     metaConnMigrationAttemptedLatency,
+			Duration:     base.DefaultHistogramWindowInterval(),
+			BucketConfig: metric.IOLatencyBuckets,
 		}),
 		ConnMigrationTransferResponseMessageSize: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaConnMigrationTransferResponseMessageSize,
-			Duration: base.DefaultHistogramWindowInterval(),
-			Buckets:  metric.DataSize16MBBuckets,
-			MaxVal:   maxExpectedTransferResponseMessageSize,
-			SigFigs:  1,
+			Metadata:     metaConnMigrationTransferResponseMessageSize,
+			Duration:     base.DefaultHistogramWindowInterval(),
+			BucketConfig: metric.DataSize16MBBuckets,
+			MaxVal:       maxExpectedTransferResponseMessageSize,
+			SigFigs:      1,
 		}),
 		QueryCancelReceivedPGWire: metric.NewCounter(metaQueryCancelReceivedPGWire),
 		QueryCancelReceivedHTTP:   metric.NewCounter(metaQueryCancelReceivedHTTP),

pkg/ccl/streamingccl/streamingest/metrics.go

@@ -163,25 +163,25 @@ func MakeMetrics(histogramWindow time.Duration) metric.Struct {
 		ResolvedEvents:       metric.NewCounter(metaReplicationResolvedEventsIngested),
 		JobProgressUpdates:   metric.NewCounter(metaJobProgressUpdates),
 		FlushHistNanos: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaReplicationFlushHistNanos,
-			Duration: histogramWindow,
-			Buckets:  metric.BatchProcessLatencyBuckets,
-			MaxVal:   streamingFlushHistMaxLatency.Nanoseconds(),
-			SigFigs:  1,
+			Metadata:     metaReplicationFlushHistNanos,
+			Duration:     histogramWindow,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
+			MaxVal:       streamingFlushHistMaxLatency.Nanoseconds(),
+			SigFigs:      1,
 		}),
 		CommitLatency: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaReplicationCommitLatency,
-			Duration: histogramWindow,
-			Buckets:  metric.BatchProcessLatencyBuckets,
-			MaxVal:   streamingCommitLatencyMaxValue.Nanoseconds(),
-			SigFigs:  1,
+			Metadata:     metaReplicationCommitLatency,
+			Duration:     histogramWindow,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
+			MaxVal:       streamingCommitLatencyMaxValue.Nanoseconds(),
+			SigFigs:      1,
 		}),
 		AdmitLatency: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaReplicationAdmitLatency,
-			Duration: histogramWindow,
-			Buckets:  metric.BatchProcessLatencyBuckets,
-			MaxVal:   streamingAdmitLatencyMaxValue.Nanoseconds(),
-			SigFigs:  1,
+			Metadata:     metaReplicationAdmitLatency,
+			Duration:     histogramWindow,
+			BucketConfig: metric.BatchProcessLatencyBuckets,
+			MaxVal:       streamingAdmitLatencyMaxValue.Nanoseconds(),
+			SigFigs:      1,
 		}),
 		RunningCount:                metric.NewGauge(metaStreamsRunning),
 		EarliestDataCheckpointSpan:  metric.NewGauge(metaEarliestDataCheckpointSpan),

pkg/kv/bulk/bulk_metrics.go

@@ -52,11 +52,11 @@ const log10int64times1000 = 19 * 1000
 func MakeBulkMetrics(histogramWindow time.Duration) Metrics {
 	return Metrics{
 		MaxBytesHist: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaMemMaxBytes,
-			Duration: histogramWindow,
-			MaxVal:   log10int64times1000,
-			SigFigs:  3,
-			Buckets:  metric.MemoryUsage64MBBuckets,
+			Metadata:     metaMemMaxBytes,
+			Duration:     histogramWindow,
+			MaxVal:       log10int64times1000,
+			SigFigs:      3,
+			BucketConfig: metric.MemoryUsage64MBBuckets,
 		}),
 		CurBytesCount: metric.NewGauge(metaMemCurBytes),
 	}

pkg/kv/kvclient/kvcoord/txn_metrics.go

@@ -269,20 +269,20 @@ func MakeTxnMetrics(histogramWindow time.Duration) TxnMetrics {
 		RefreshMemoryLimitExceeded:    metric.NewCounter(metaRefreshMemoryLimitExceeded),
 		RefreshAutoRetries:            metric.NewCounter(metaRefreshAutoRetries),
 		Durations: metric.NewHistogram(metric.HistogramOptions{
-			Mode:     metric.HistogramModePreferHdrLatency,
-			Metadata: metaDurationsHistograms,
-			Duration: histogramWindow,
-			Buckets:  metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePreferHdrLatency,
+			Metadata:     metaDurationsHistograms,
+			Duration:     histogramWindow,
+			BucketConfig: metric.IOLatencyBuckets,
 		}),
 		TxnsWithCondensedIntents:      metric.NewCounter(metaTxnsWithCondensedIntentSpans),
 		TxnsWithCondensedIntentsGauge: metric.NewGauge(metaTxnsWithCondensedIntentSpansGauge),
 		TxnsRejectedByLockSpanBudget:  metric.NewCounter(metaTxnsRejectedByLockSpanBudget),
 		Restarts: metric.NewHistogram(metric.HistogramOptions{
-			Metadata: metaRestartsHistogram,
-			Duration: histogramWindow,
-			MaxVal:   100,
-			SigFigs:  3,
-			Buckets:  metric.Count1KBuckets,
+			Metadata:     metaRestartsHistogram,
+			Duration:     histogramWindow,
+			MaxVal:       100,
+			SigFigs:      3,
+			BucketConfig: metric.Count1KBuckets,
 		}),
 		RestartsWriteTooOld:            telemetry.NewCounterWithMetric(metaRestartsWriteTooOld),
 		RestartsWriteTooOldMulti:       telemetry.NewCounterWithMetric(metaRestartsWriteTooOldMulti),

pkg/kv/kvprober/kvprober.go

@@ -277,18 +277,18 @@ func NewProber(opts Opts) *Prober {
 			ReadProbeAttempts: metric.NewCounter(metaReadProbeAttempts),
 			ReadProbeFailures: metric.NewCounter(metaReadProbeFailures),
 			ReadProbeLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePreferHdrLatency,
-				Metadata: metaReadProbeLatency,
-				Duration: opts.HistogramWindowInterval,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePreferHdrLatency,
+				Metadata:     metaReadProbeLatency,
+				Duration:     opts.HistogramWindowInterval,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 			WriteProbeAttempts: metric.NewCounter(metaWriteProbeAttempts),
 			WriteProbeFailures: metric.NewCounter(metaWriteProbeFailures),
 			WriteProbeLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePreferHdrLatency,
-				Metadata: metaWriteProbeLatency,
-				Duration: opts.HistogramWindowInterval,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePreferHdrLatency,
+				Metadata:     metaWriteProbeLatency,
+				Duration:     opts.HistogramWindowInterval,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 			WriteProbeQuarantineOldestDuration: metric.NewFunctionalGauge(
 				metaWriteProbeQuarantineOldestDuration,

pkg/kv/kvserver/kvflowcontrol/kvflowcontroller/kvflowcontroller_metrics.go

@@ -183,10 +183,10 @@ func newMetrics(c *Controller) *metrics {
 		)
 		m.WaitDuration[wc] = metric.NewHistogram(
 			metric.HistogramOptions{
-				Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration),
-				Duration: base.DefaultHistogramWindowInterval(),
-				Buckets:  metric.IOLatencyBuckets,
-				Mode:     metric.HistogramModePrometheus,
+				Metadata:     annotateMetricTemplateWithWorkClass(wc, waitDuration),
+				Duration:     base.DefaultHistogramWindowInterval(),
+				BucketConfig: metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePrometheus,
 			},
 		)
 		m.TotalStreamCount[wc] = metric.NewFunctionalGauge(

pkg/kv/kvserver/kvflowcontrol/kvflowhandle/kvflowhandle_metrics.go

@@ -89,10 +89,10 @@ func NewMetrics(registry *metric.Registry) *Metrics {
 		)
 		m.WaitDuration[wc] = metric.NewHistogram(
 			metric.HistogramOptions{
-				Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration),
-				Duration: base.DefaultHistogramWindowInterval(),
-				Buckets:  metric.IOLatencyBuckets,
-				Mode:     metric.HistogramModePrometheus,
+				Metadata:     annotateMetricTemplateWithWorkClass(wc, waitDuration),
+				Duration:     base.DefaultHistogramWindowInterval(),
+				BucketConfig: metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePrometheus,
 			},
 		)
 	}

pkg/kv/kvserver/liveness/liveness.go

@@ -330,10 +330,10 @@ func NewNodeLiveness(opts NodeLivenessOptions) *NodeLiveness {
 		HeartbeatFailures:  telemetry.NewCounterWithMetric(metaHeartbeatFailures),
 		EpochIncrements:    telemetry.NewCounterWithMetric(metaEpochIncrements),
 		HeartbeatLatency: metric.NewHistogram(metric.HistogramOptions{
-			Mode:     metric.HistogramModePreferHdrLatency,
-			Metadata: metaHeartbeatLatency,
-			Duration: opts.HistogramWindowInterval,
-			Buckets:  metric.IOLatencyBuckets,
+			Mode:         metric.HistogramModePreferHdrLatency,
+			Metadata:     metaHeartbeatLatency,
+			Duration:     opts.HistogramWindowInterval,
+			BucketConfig: metric.IOLatencyBuckets,
 		}),
 	}
 	nl.mu.nodes = make(map[roachpb.NodeID]Record)

pkg/kv/kvserver/logstore/logstore_bench_test.go

@@ -68,10 +68,10 @@ func runBenchmarkLogStore_StoreEntries(b *testing.B, bytes int64) {
 		Settings:    st,
 		Metrics: Metrics{
 			RaftLogCommitLatency: metric.NewHistogram(metric.HistogramOptions{
-				Mode:     metric.HistogramModePrometheus,
-				Metadata: metric.Metadata{},
-				Duration: 10 * time.Second,
-				Buckets:  metric.IOLatencyBuckets,
+				Mode:         metric.HistogramModePrometheus,
+				Metadata:     metric.Metadata{},
+				Duration:     10 * time.Second,
+				BucketConfig: metric.IOLatencyBuckets,
 			}),
 		},
 	}