From d8524b24470f7408b1f58a733a4f0585527d4358 Mon Sep 17 00:00:00 2001 From: Eric Harmeling Date: Wed, 26 Jul 2023 14:06:23 -0400 Subject: [PATCH] add StaticBucketConfig to HistogramOptions --- pkg/ccl/changefeedccl/metrics.go | 86 +++++----- pkg/ccl/sqlproxyccl/connector_test.go | 24 +-- pkg/ccl/sqlproxyccl/metrics.go | 34 ++-- pkg/ccl/streamingccl/streamingest/metrics.go | 30 ++-- pkg/kv/bulk/bulk_metrics.go | 10 +- pkg/kv/kvclient/kvcoord/txn_metrics.go | 18 +-- pkg/kv/kvprober/kvprober.go | 16 +- .../kvserver/client_manual_proposal_test.go | 8 +- .../kvflowcontroller_metrics.go | 8 +- .../kvflowhandle/kvflowhandle_metrics.go | 8 +- pkg/kv/kvserver/liveness/liveness.go | 8 +- .../kvserver/logstore/logstore_bench_test.go | 8 +- pkg/kv/kvserver/metrics.go | 88 ++++++----- pkg/kv/kvserver/txnwait/metrics.go | 16 +- pkg/rpc/clock_offset.go | 2 +- pkg/server/node.go | 8 +- pkg/sql/conn_executor.go | 76 ++++----- pkg/sql/execinfra/metrics.go | 20 +-- pkg/sql/mem_metrics.go | 10 +- pkg/sql/pgwire/pre_serve.go | 8 +- pkg/sql/pgwire/server.go | 8 +- pkg/sql/sqlstats/sslocal/sql_stats_test.go | 10 +- pkg/sql/ttl/ttljob/ttljob_metrics.go | 24 +-- pkg/util/admission/elastic_cpu_granter.go | 8 +- pkg/util/admission/work_queue.go | 8 +- pkg/util/metric/aggmetric/agg_metric_test.go | 18 +-- pkg/util/metric/histogram_buckets.go | 147 ++++++++++-------- pkg/util/metric/histogram_buckets_test.go | 8 +- pkg/util/metric/metric.go | 15 +- pkg/util/metric/metric_test.go | 14 +- pkg/util/metric/registry_test.go | 24 +-- 31 files changed, 397 insertions(+), 373 deletions(-) diff --git a/pkg/ccl/changefeedccl/metrics.go b/pkg/ccl/changefeedccl/metrics.go index 060ea6d9bcea..0b1810d77a33 100644 --- a/pkg/ccl/changefeedccl/metrics.go +++ b/pkg/ccl/changefeedccl/metrics.go @@ -552,52 +552,52 @@ func newAggregateMetrics(histogramWindow time.Duration) *AggMetrics { EmittedMessages: b.Counter(metaChangefeedEmittedMessages), FilteredMessages: b.Counter(metaChangefeedFilteredMessages), MessageSize: b.Histogram(metric.HistogramOptions{ - Metadata: metaMessageSize, - Duration: histogramWindow, - MaxVal: 10 << 20, /* 10MB max message size */ - SigFigs: 1, - Buckets: metric.DataSize16MBBuckets, + Metadata: metaMessageSize, + Duration: histogramWindow, + MaxVal: 10 << 20, /* 10MB max message size */ + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.DataSize16MBBuckets], }), EmittedBytes: b.Counter(metaChangefeedEmittedBytes), FlushedBytes: b.Counter(metaChangefeedFlushedBytes), Flushes: b.Counter(metaChangefeedFlushes), SizeBasedFlushes: b.Counter(metaSizeBasedFlushes), ParallelIOQueueNanos: b.Histogram(metric.HistogramOptions{ - Metadata: metaChangefeedParallelIOQueueNanos, - Duration: histogramWindow, - MaxVal: changefeedIOQueueMaxLatency.Nanoseconds(), - SigFigs: 2, - Buckets: metric.BatchProcessLatencyBuckets, + Metadata: metaChangefeedParallelIOQueueNanos, + Duration: histogramWindow, + MaxVal: changefeedIOQueueMaxLatency.Nanoseconds(), + SigFigs: 2, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), SinkIOInflight: b.Gauge(metaChangefeedSinkIOInflight), BatchHistNanos: b.Histogram(metric.HistogramOptions{ - Metadata: metaChangefeedBatchHistNanos, - Duration: histogramWindow, - MaxVal: changefeedBatchHistMaxLatency.Nanoseconds(), - SigFigs: 1, - Buckets: metric.BatchProcessLatencyBuckets, + Metadata: metaChangefeedBatchHistNanos, + Duration: histogramWindow, + MaxVal: changefeedBatchHistMaxLatency.Nanoseconds(), + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), FlushHistNanos: b.Histogram(metric.HistogramOptions{ - Metadata: metaChangefeedFlushHistNanos, - Duration: histogramWindow, - MaxVal: changefeedFlushHistMaxLatency.Nanoseconds(), - SigFigs: 2, - Buckets: metric.BatchProcessLatencyBuckets, + Metadata: metaChangefeedFlushHistNanos, + Duration: histogramWindow, + MaxVal: changefeedFlushHistMaxLatency.Nanoseconds(), + SigFigs: 2, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), CommitLatency: b.Histogram(metric.HistogramOptions{ - Metadata: metaCommitLatency, - Duration: histogramWindow, - MaxVal: commitLatencyMaxValue.Nanoseconds(), - SigFigs: 1, - Buckets: metric.BatchProcessLatencyBuckets, + Metadata: metaCommitLatency, + Duration: histogramWindow, + MaxVal: commitLatencyMaxValue.Nanoseconds(), + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), AdmitLatency: b.Histogram(metric.HistogramOptions{ - Metadata: metaAdmitLatency, - Duration: histogramWindow, - MaxVal: admitLatencyMaxValue.Nanoseconds(), - SigFigs: 1, - Buckets: metric.BatchProcessLatencyBuckets, + Metadata: metaAdmitLatency, + Duration: histogramWindow, + MaxVal: admitLatencyMaxValue.Nanoseconds(), + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), BackfillCount: b.Gauge(metaChangefeedBackfillCount), BackfillPendingRanges: b.Gauge(metaChangefeedBackfillPendingRanges), @@ -712,27 +712,27 @@ func MakeMetrics(histogramWindow time.Duration) metric.Struct { Failures: metric.NewCounter(metaChangefeedFailures), QueueTimeNanos: metric.NewCounter(metaEventQueueTime), CheckpointHistNanos: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaChangefeedCheckpointHistNanos, - Duration: histogramWindow, - MaxVal: changefeedCheckpointHistMaxLatency.Nanoseconds(), - SigFigs: 2, - Buckets: metric.IOLatencyBuckets, + Metadata: metaChangefeedCheckpointHistNanos, + Duration: histogramWindow, + MaxVal: changefeedCheckpointHistMaxLatency.Nanoseconds(), + SigFigs: 2, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), FrontierUpdates: metric.NewCounter(metaChangefeedFrontierUpdates), ThrottleMetrics: cdcutils.MakeMetrics(histogramWindow), // Below two metrics were never implemented using the hdr histogram. Set ForceUsePrometheus // to true. ParallelConsumerFlushNanos: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaChangefeedEventConsumerFlushNanos, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, - Mode: metric.HistogramModePrometheus, + Metadata: metaChangefeedEventConsumerFlushNanos, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], + Mode: metric.HistogramModePrometheus, }), ParallelConsumerConsumeNanos: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaChangefeedEventConsumerConsumeNanos, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, - Mode: metric.HistogramModePrometheus, + Metadata: metaChangefeedEventConsumerConsumeNanos, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], + Mode: metric.HistogramModePrometheus, }), ParallelConsumerInFlightEvents: metric.NewGauge(metaChangefeedEventConsumerInFlightEvents), } diff --git a/pkg/ccl/sqlproxyccl/connector_test.go b/pkg/ccl/sqlproxyccl/connector_test.go index fbbeb968d615..4a26505261db 100644 --- a/pkg/ccl/sqlproxyccl/connector_test.go +++ b/pkg/ccl/sqlproxyccl/connector_test.go @@ -381,10 +381,10 @@ func TestConnector_dialTenantCluster(t *testing.T) { c := &connector{ TenantID: roachpb.MustMakeTenantID(42), DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePrometheus, - Metadata: metaDialTenantLatency, - Duration: time.Millisecond, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePrometheus, + Metadata: metaDialTenantLatency, + Duration: time.Millisecond, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), DialTenantRetries: metric.NewCounter(metaDialTenantRetries), } @@ -466,10 +466,10 @@ func TestConnector_dialTenantCluster(t *testing.T) { c := &connector{ DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaDialTenantLatency, - Duration: time.Millisecond, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaDialTenantLatency, + Duration: time.Millisecond, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), DialTenantRetries: metric.NewCounter(metaDialTenantRetries), } @@ -500,10 +500,10 @@ func TestConnector_dialTenantCluster(t *testing.T) { c := &connector{ TenantID: roachpb.MustMakeTenantID(42), DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaDialTenantLatency, - Duration: time.Millisecond, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaDialTenantLatency, + Duration: time.Millisecond, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), DialTenantRetries: metric.NewCounter(metaDialTenantRetries), } diff --git a/pkg/ccl/sqlproxyccl/metrics.go b/pkg/ccl/sqlproxyccl/metrics.go index 28d9ad426847..3b21933a0860 100644 --- a/pkg/ccl/sqlproxyccl/metrics.go +++ b/pkg/ccl/sqlproxyccl/metrics.go @@ -234,19 +234,19 @@ func makeProxyMetrics() metrics { RefusedConnCount: metric.NewCounter(metaRefusedConnCount), SuccessfulConnCount: metric.NewCounter(metaSuccessfulConnCount), ConnectionLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaConnMigrationAttemptedCount, - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaConnMigrationAttemptedCount, + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), AuthFailedCount: metric.NewCounter(metaAuthFailedCount), ExpiredClientConnCount: metric.NewCounter(metaExpiredClientConnCount), // Connector metrics. DialTenantLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaDialTenantLatency, - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets}, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaDialTenantLatency, + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets]}, ), DialTenantRetries: metric.NewCounter(metaDialTenantRetries), // Connection migration metrics. @@ -255,17 +255,17 @@ func makeProxyMetrics() metrics { ConnMigrationErrorRecoverableCount: metric.NewCounter(metaConnMigrationErrorRecoverableCount), ConnMigrationAttemptedCount: metric.NewCounter(metaConnMigrationAttemptedCount), ConnMigrationAttemptedLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaConnMigrationAttemptedLatency, - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaConnMigrationAttemptedLatency, + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), ConnMigrationTransferResponseMessageSize: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaConnMigrationTransferResponseMessageSize, - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.DataSize16MBBuckets, - MaxVal: maxExpectedTransferResponseMessageSize, - SigFigs: 1, + Metadata: metaConnMigrationTransferResponseMessageSize, + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.DataSize16MBBuckets], + MaxVal: maxExpectedTransferResponseMessageSize, + SigFigs: 1, }), QueryCancelReceivedPGWire: metric.NewCounter(metaQueryCancelReceivedPGWire), QueryCancelReceivedHTTP: metric.NewCounter(metaQueryCancelReceivedHTTP), diff --git a/pkg/ccl/streamingccl/streamingest/metrics.go b/pkg/ccl/streamingccl/streamingest/metrics.go index 6104150333ac..286498f8378c 100644 --- a/pkg/ccl/streamingccl/streamingest/metrics.go +++ b/pkg/ccl/streamingccl/streamingest/metrics.go @@ -171,25 +171,25 @@ func MakeMetrics(histogramWindow time.Duration) metric.Struct { JobProgressUpdates: metric.NewCounter(metaJobProgressUpdates), ReplanCount: metric.NewCounter(metaDistSQLReplanCount), FlushHistNanos: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaReplicationFlushHistNanos, - Duration: histogramWindow, - Buckets: metric.BatchProcessLatencyBuckets, - MaxVal: streamingFlushHistMaxLatency.Nanoseconds(), - SigFigs: 1, + Metadata: metaReplicationFlushHistNanos, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], + MaxVal: streamingFlushHistMaxLatency.Nanoseconds(), + SigFigs: 1, }), CommitLatency: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaReplicationCommitLatency, - Duration: histogramWindow, - Buckets: metric.BatchProcessLatencyBuckets, - MaxVal: streamingCommitLatencyMaxValue.Nanoseconds(), - SigFigs: 1, + Metadata: metaReplicationCommitLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], + MaxVal: streamingCommitLatencyMaxValue.Nanoseconds(), + SigFigs: 1, }), AdmitLatency: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaReplicationAdmitLatency, - Duration: histogramWindow, - Buckets: metric.BatchProcessLatencyBuckets, - MaxVal: streamingAdmitLatencyMaxValue.Nanoseconds(), - SigFigs: 1, + Metadata: metaReplicationAdmitLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], + MaxVal: streamingAdmitLatencyMaxValue.Nanoseconds(), + SigFigs: 1, }), RunningCount: metric.NewGauge(metaStreamsRunning), EarliestDataCheckpointSpan: metric.NewGauge(metaEarliestDataCheckpointSpan), diff --git a/pkg/kv/bulk/bulk_metrics.go b/pkg/kv/bulk/bulk_metrics.go index 7cbbc748a20b..42cb0cac271f 100644 --- a/pkg/kv/bulk/bulk_metrics.go +++ b/pkg/kv/bulk/bulk_metrics.go @@ -52,11 +52,11 @@ const log10int64times1000 = 19 * 1000 func MakeBulkMetrics(histogramWindow time.Duration) Metrics { return Metrics{ MaxBytesHist: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaMemMaxBytes, - Duration: histogramWindow, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: metaMemMaxBytes, + Duration: histogramWindow, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }), CurBytesCount: metric.NewGauge(metaMemCurBytes), } diff --git a/pkg/kv/kvclient/kvcoord/txn_metrics.go b/pkg/kv/kvclient/kvcoord/txn_metrics.go index e35abeadd493..b8503ee11f85 100644 --- a/pkg/kv/kvclient/kvcoord/txn_metrics.go +++ b/pkg/kv/kvclient/kvcoord/txn_metrics.go @@ -285,20 +285,20 @@ func MakeTxnMetrics(histogramWindow time.Duration) TxnMetrics { ClientRefreshAutoRetries: metric.NewCounter(metaClientRefreshAutoRetries), ServerRefreshSuccess: metric.NewCounter(metaServerRefreshSuccess), Durations: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaDurationsHistograms, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaDurationsHistograms, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), TxnsWithCondensedIntents: metric.NewCounter(metaTxnsWithCondensedIntentSpans), TxnsWithCondensedIntentsGauge: metric.NewGauge(metaTxnsWithCondensedIntentSpansGauge), TxnsRejectedByLockSpanBudget: metric.NewCounter(metaTxnsRejectedByLockSpanBudget), Restarts: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaRestartsHistogram, - Duration: histogramWindow, - MaxVal: 100, - SigFigs: 3, - Buckets: metric.Count1KBuckets, + Metadata: metaRestartsHistogram, + Duration: histogramWindow, + MaxVal: 100, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.Count1KBuckets], }), RestartsWriteTooOld: telemetry.NewCounterWithMetric(metaRestartsWriteTooOld), RestartsWriteTooOldMulti: telemetry.NewCounterWithMetric(metaRestartsWriteTooOldMulti), diff --git a/pkg/kv/kvprober/kvprober.go b/pkg/kv/kvprober/kvprober.go index 3346f67df803..749ea046e375 100644 --- a/pkg/kv/kvprober/kvprober.go +++ b/pkg/kv/kvprober/kvprober.go @@ -276,18 +276,18 @@ func NewProber(opts Opts) *Prober { ReadProbeAttempts: metric.NewCounter(metaReadProbeAttempts), ReadProbeFailures: metric.NewCounter(metaReadProbeFailures), ReadProbeLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaReadProbeLatency, - Duration: opts.HistogramWindowInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaReadProbeLatency, + Duration: opts.HistogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), WriteProbeAttempts: metric.NewCounter(metaWriteProbeAttempts), WriteProbeFailures: metric.NewCounter(metaWriteProbeFailures), WriteProbeLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaWriteProbeLatency, - Duration: opts.HistogramWindowInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaWriteProbeLatency, + Duration: opts.HistogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), WriteProbeQuarantineOldestDuration: metric.NewFunctionalGauge( metaWriteProbeQuarantineOldestDuration, diff --git a/pkg/kv/kvserver/client_manual_proposal_test.go b/pkg/kv/kvserver/client_manual_proposal_test.go index 003693d1f3fc..5637985f7dc9 100644 --- a/pkg/kv/kvserver/client_manual_proposal_test.go +++ b/pkg/kv/kvserver/client_manual_proposal_test.go @@ -232,10 +232,10 @@ LIMIT Settings: st, Metrics: logstore.Metrics{ RaftLogCommitLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePrometheus, - Metadata: fakeMeta, - Duration: time.Millisecond, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePrometheus, + Metadata: fakeMeta, + Duration: time.Millisecond, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), }, } diff --git a/pkg/kv/kvserver/kvflowcontrol/kvflowcontroller/kvflowcontroller_metrics.go b/pkg/kv/kvserver/kvflowcontrol/kvflowcontroller/kvflowcontroller_metrics.go index 2882ce56fbc2..f6b2d60d0083 100644 --- a/pkg/kv/kvserver/kvflowcontrol/kvflowcontroller/kvflowcontroller_metrics.go +++ b/pkg/kv/kvserver/kvflowcontrol/kvflowcontroller/kvflowcontroller_metrics.go @@ -186,10 +186,10 @@ func newMetrics(c *Controller) *metrics { ) m.WaitDuration[wc] = metric.NewHistogram( metric.HistogramOptions{ - Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration), - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, - Mode: metric.HistogramModePrometheus, + Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration), + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], + Mode: metric.HistogramModePrometheus, }, ) m.TotalStreamCount[wc] = metric.NewFunctionalGauge( diff --git a/pkg/kv/kvserver/kvflowcontrol/kvflowhandle/kvflowhandle_metrics.go b/pkg/kv/kvserver/kvflowcontrol/kvflowhandle/kvflowhandle_metrics.go index 5006781b53e8..cb9f19702dc5 100644 --- a/pkg/kv/kvserver/kvflowcontrol/kvflowhandle/kvflowhandle_metrics.go +++ b/pkg/kv/kvserver/kvflowcontrol/kvflowhandle/kvflowhandle_metrics.go @@ -109,10 +109,10 @@ func NewMetrics(registry *metric.Registry) *Metrics { ) m.WaitDuration[wc] = metric.NewHistogram( metric.HistogramOptions{ - Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration), - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, - Mode: metric.HistogramModePrometheus, + Metadata: annotateMetricTemplateWithWorkClass(wc, waitDuration), + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], + Mode: metric.HistogramModePrometheus, }, ) } diff --git a/pkg/kv/kvserver/liveness/liveness.go b/pkg/kv/kvserver/liveness/liveness.go index 9450ee6a2add..802df7235b98 100644 --- a/pkg/kv/kvserver/liveness/liveness.go +++ b/pkg/kv/kvserver/liveness/liveness.go @@ -367,10 +367,10 @@ func NewNodeLiveness(opts NodeLivenessOptions) *NodeLiveness { HeartbeatFailures: telemetry.NewCounterWithMetric(metaHeartbeatFailures), EpochIncrements: telemetry.NewCounterWithMetric(metaEpochIncrements), HeartbeatLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaHeartbeatLatency, - Duration: opts.HistogramWindowInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaHeartbeatLatency, + Duration: opts.HistogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), } nl.cache = newCache(opts.Gossip, opts.Clock, nl.cacheUpdated) diff --git a/pkg/kv/kvserver/logstore/logstore_bench_test.go b/pkg/kv/kvserver/logstore/logstore_bench_test.go index d318d1b7852c..e9f6c2f82b5f 100644 --- a/pkg/kv/kvserver/logstore/logstore_bench_test.go +++ b/pkg/kv/kvserver/logstore/logstore_bench_test.go @@ -68,10 +68,10 @@ func runBenchmarkLogStore_StoreEntries(b *testing.B, bytes int64) { Settings: st, Metrics: Metrics{ RaftLogCommitLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePrometheus, - Metadata: metric.Metadata{}, - Duration: 10 * time.Second, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePrometheus, + Metadata: metric.Metadata{}, + Duration: 10 * time.Second, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), }, } diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index 5730d55de188..732497e3b755 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -2838,10 +2838,10 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { LeaseRequestSuccessCount: metric.NewCounter(metaLeaseRequestSuccessCount), LeaseRequestErrorCount: metric.NewCounter(metaLeaseRequestErrorCount), LeaseRequestLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaLeaseRequestLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaLeaseRequestLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), LeaseTransferSuccessCount: metric.NewCounter(metaLeaseTransferSuccessCount), LeaseTransferErrorCount: metric.NewCounter(metaLeaseTransferErrorCount), @@ -2871,12 +2871,14 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { AverageCPUNanosPerSecond: metric.NewGaugeFloat64(metaAverageCPUNanosPerSecond), RecentReplicaCPUNanosPerSecond: metric.NewManualWindowHistogram( metaRecentReplicaCPUNanosPerSecond, - metric.ReplicaCPUTimeBuckets, + metric.StaticBucketConfigs[metric.ReplicaCPUTimeBuckets]. + GetBucketsFromBucketConfig(), true, /* withRotate */ ), RecentReplicaQueriesPerSecond: metric.NewManualWindowHistogram( metaRecentReplicaQueriesPerSecond, - metric.ReplicaBatchRequestCountBuckets, + metric.StaticBucketConfigs[metric.ReplicaBatchRequestCountBuckets]. + GetBucketsFromBucketConfig(), true, /* withRotate */ ), @@ -3008,50 +3010,50 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { RaftProposalsDropped: metric.NewCounter(metaRaftProposalsDropped), RaftProposalsDroppedLeader: metric.NewCounter(metaRaftProposalsDroppedLeader), RaftQuotaPoolPercentUsed: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaRaftQuotaPoolPercentUsed, - Duration: histogramWindow, - MaxVal: 100, - SigFigs: 1, - Buckets: metric.Percent100Buckets, + Metadata: metaRaftQuotaPoolPercentUsed, + Duration: histogramWindow, + MaxVal: 100, + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.Percent100Buckets], }), RaftWorkingDurationNanos: metric.NewCounter(metaRaftWorkingDurationNanos), RaftTickingDurationNanos: metric.NewCounter(metaRaftTickingDurationNanos), RaftCommandsApplied: metric.NewCounter(metaRaftCommandsApplied), RaftLogCommitLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaRaftLogCommitLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaRaftLogCommitLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftCommandCommitLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaRaftCommandCommitLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaRaftCommandCommitLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftHandleReadyLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaRaftHandleReadyLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaRaftHandleReadyLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftApplyCommittedLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaRaftApplyCommittedLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaRaftApplyCommittedLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftReplicationLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePrometheus, - Metadata: metaRaftReplicationLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePrometheus, + Metadata: metaRaftReplicationLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftSchedulerLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaRaftSchedulerLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaRaftSchedulerLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), RaftTimeoutCampaign: metric.NewCounter(metaRaftTimeoutCampaign), RaftStorageReadBytes: metric.NewCounter(metaRaftStorageReadBytes), @@ -3201,16 +3203,16 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { // Replica batch evaluation. ReplicaReadBatchEvaluationLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaReplicaReadBatchEvaluationLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaReplicaReadBatchEvaluationLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), ReplicaWriteBatchEvaluationLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaReplicaWriteBatchEvaluationLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaReplicaWriteBatchEvaluationLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), FlushUtilization: metric.NewGaugeFloat64(metaStorageFlushUtilization), FsyncLatency: metric.NewManualWindowHistogram( diff --git a/pkg/kv/kvserver/txnwait/metrics.go b/pkg/kv/kvserver/txnwait/metrics.go index 4610fd8e375c..4321ee27a712 100644 --- a/pkg/kv/kvserver/txnwait/metrics.go +++ b/pkg/kv/kvserver/txnwait/metrics.go @@ -73,10 +73,10 @@ func NewMetrics(histogramWindowInterval time.Duration) *Metrics { Measurement: "Pusher wait time", Unit: metric.Unit_NANOSECONDS, }, - MaxVal: time.Hour.Nanoseconds(), - SigFigs: 1, - Duration: histogramWindowInterval, - Buckets: metric.LongRunning60mLatencyBuckets, + MaxVal: time.Hour.Nanoseconds(), + SigFigs: 1, + Duration: histogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.LongRunning60mLatencyBuckets], }), QueryWaitTime: metric.NewHistogram(metric.HistogramOptions{ @@ -86,10 +86,10 @@ func NewMetrics(histogramWindowInterval time.Duration) *Metrics { Measurement: "Query wait time", Unit: metric.Unit_NANOSECONDS, }, - MaxVal: time.Hour.Nanoseconds(), - SigFigs: 1, - Duration: histogramWindowInterval, - Buckets: metric.LongRunning60mLatencyBuckets, + MaxVal: time.Hour.Nanoseconds(), + SigFigs: 1, + Duration: histogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.LongRunning60mLatencyBuckets], }), DeadlocksTotal: metric.NewCounter( diff --git a/pkg/rpc/clock_offset.go b/pkg/rpc/clock_offset.go index 66f0cb4ec53b..55f46752090d 100644 --- a/pkg/rpc/clock_offset.go +++ b/pkg/rpc/clock_offset.go @@ -157,7 +157,7 @@ func newRemoteClockMonitor( // NB: the choice of IO over Network buckets is somewhat debatable, but // it's fine. Heartbeats can take >1s which the IO buckets can represent, // but the Network buckets top out at 1s. - Buckets: metric.IOLatencyBuckets, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), } return &r diff --git a/pkg/server/node.go b/pkg/server/node.go index 9a9dd08e55e0..942f3a307ae6 100644 --- a/pkg/server/node.go +++ b/pkg/server/node.go @@ -276,10 +276,10 @@ type nodeMetrics struct { func makeNodeMetrics(reg *metric.Registry, histogramWindow time.Duration) nodeMetrics { nm := nodeMetrics{ Latency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: metaExecLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: metaExecLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), Success: metric.NewCounter(metaExecSuccess), Err: metric.NewCounter(metaExecError), diff --git a/pkg/sql/conn_executor.go b/pkg/sql/conn_executor.go index b43acb8d094c..b17371e1ecdb 100644 --- a/pkg/sql/conn_executor.go +++ b/pkg/sql/conn_executor.go @@ -505,34 +505,34 @@ func makeMetrics(internal bool) Metrics { SQLOptPlanCacheMisses: metric.NewCounter(getMetricMeta(MetaSQLOptPlanCacheMisses, internal)), // TODO(mrtracy): See HistogramWindowInterval in server/config.go for the 6x factor. DistSQLExecLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: getMetricMeta(MetaDistSQLExecLatency, internal), - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: getMetricMeta(MetaDistSQLExecLatency, internal), + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), SQLExecLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: getMetricMeta(MetaSQLExecLatency, internal), - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: getMetricMeta(MetaSQLExecLatency, internal), + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), DistSQLServiceLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: getMetricMeta(MetaDistSQLServiceLatency, internal), - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: getMetricMeta(MetaDistSQLServiceLatency, internal), + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), SQLServiceLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: getMetricMeta(MetaSQLServiceLatency, internal), - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: getMetricMeta(MetaSQLServiceLatency, internal), + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), SQLTxnLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: getMetricMeta(MetaSQLTxnLatency, internal), - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: getMetricMeta(MetaSQLTxnLatency, internal), + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), SQLTxnsOpen: metric.NewGauge(getMetricMeta(MetaSQLTxnsOpen, internal)), SQLActiveStatements: metric.NewGauge(getMetricMeta(MetaSQLActiveQueries, internal)), @@ -558,36 +558,36 @@ func makeServerMetrics(cfg *ExecutorConfig) ServerMetrics { return ServerMetrics{ StatsMetrics: StatsMetrics{ SQLStatsMemoryMaxBytesHist: metric.NewHistogram(metric.HistogramOptions{ - Metadata: MetaSQLStatsMemMaxBytes, - Duration: cfg.HistogramWindowInterval, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: MetaSQLStatsMemMaxBytes, + Duration: cfg.HistogramWindowInterval, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }), SQLStatsMemoryCurBytesCount: metric.NewGauge(MetaSQLStatsMemCurBytes), ReportedSQLStatsMemoryMaxBytesHist: metric.NewHistogram(metric.HistogramOptions{ - Metadata: MetaReportedSQLStatsMemMaxBytes, - Duration: cfg.HistogramWindowInterval, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: MetaReportedSQLStatsMemMaxBytes, + Duration: cfg.HistogramWindowInterval, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }), ReportedSQLStatsMemoryCurBytesCount: metric.NewGauge(MetaReportedSQLStatsMemCurBytes), DiscardedStatsCount: metric.NewCounter(MetaDiscardedSQLStats), SQLStatsFlushStarted: metric.NewCounter(MetaSQLStatsFlushStarted), SQLStatsFlushFailure: metric.NewCounter(MetaSQLStatsFlushFailure), SQLStatsFlushDuration: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: MetaSQLStatsFlushDuration, - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: MetaSQLStatsFlushDuration, + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), SQLStatsRemovedRows: metric.NewCounter(MetaSQLStatsRemovedRows), SQLTxnStatsCollectionOverhead: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: MetaSQLTxnStatsCollectionOverhead, - Duration: 6 * metricsSampleInterval, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: MetaSQLTxnStatsCollectionOverhead, + Duration: 6 * metricsSampleInterval, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), }, ContentionSubsystemMetrics: txnidcache.NewMetrics(), diff --git a/pkg/sql/execinfra/metrics.go b/pkg/sql/execinfra/metrics.go index 30123ce27ee6..350886485f0f 100644 --- a/pkg/sql/execinfra/metrics.go +++ b/pkg/sql/execinfra/metrics.go @@ -133,21 +133,21 @@ func MakeDistSQLMetrics(histogramWindow time.Duration) DistSQLMetrics { FlowsActive: metric.NewGauge(metaFlowsActive), FlowsTotal: metric.NewCounter(metaFlowsTotal), MaxBytesHist: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaMemMaxBytes, - Duration: histogramWindow, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: metaMemMaxBytes, + Duration: histogramWindow, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }), CurBytesCount: metric.NewGauge(metaMemCurBytes), VecOpenFDs: metric.NewGauge(metaVecOpenFDs), CurDiskBytesCount: metric.NewGauge(metaDiskCurBytes), MaxDiskBytesHist: metric.NewHistogram(metric.HistogramOptions{ - Metadata: metaDiskMaxBytes, - Duration: histogramWindow, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets}), + Metadata: metaDiskMaxBytes, + Duration: histogramWindow, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets]}), QueriesSpilled: metric.NewCounter(metaQueriesSpilled), SpilledBytesWritten: metric.NewCounter(metaSpilledBytesWritten), SpilledBytesRead: metric.NewCounter(metaSpilledBytesRead), diff --git a/pkg/sql/mem_metrics.go b/pkg/sql/mem_metrics.go index 995a95d1226d..2be7620017f4 100644 --- a/pkg/sql/mem_metrics.go +++ b/pkg/sql/mem_metrics.go @@ -74,11 +74,11 @@ func makeMemMetricHistogram( metadata metric.Metadata, histogramWindow time.Duration, ) metric.IHistogram { return metric.NewHistogram(metric.HistogramOptions{ - Metadata: metadata, - Duration: histogramWindow, - MaxVal: log10int64times1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: metadata, + Duration: histogramWindow, + MaxVal: log10int64times1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }) } diff --git a/pkg/sql/pgwire/pre_serve.go b/pkg/sql/pgwire/pre_serve.go index 37b9ae56281a..63bf9f54bf5a 100644 --- a/pkg/sql/pgwire/pre_serve.go +++ b/pkg/sql/pgwire/pre_serve.go @@ -192,10 +192,10 @@ func makeTenantIndependentMetrics(histogramWindow time.Duration) tenantIndepende PreServeNewConns: metric.NewCounter(MetaPreServeNewConns), PreServeConnFailures: metric.NewCounter(MetaPreServeConnFailures), PreServeMaxBytes: metric.NewHistogram(metric.HistogramOptions{ - Metadata: MetaPreServeMaxBytes, - Duration: histogramWindow, - Buckets: metric.MemoryUsage64MBBuckets, - Mode: metric.HistogramModePrometheus, + Metadata: MetaPreServeMaxBytes, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], + Mode: metric.HistogramModePrometheus, }), PreServeCurBytes: metric.NewGauge(MetaPreServeCurBytes), } diff --git a/pkg/sql/pgwire/server.go b/pkg/sql/pgwire/server.go index 391d2c08a313..a2ffb9ce7f9a 100644 --- a/pkg/sql/pgwire/server.go +++ b/pkg/sql/pgwire/server.go @@ -279,10 +279,10 @@ func newTenantSpecificMetrics( NewConns: metric.NewCounter(MetaNewConns), ConnsWaitingToHash: metric.NewGauge(MetaConnsWaitingToHash), ConnLatency: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: MetaConnLatency, - Duration: histogramWindow, - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: MetaConnLatency, + Duration: histogramWindow, + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), ConnFailures: metric.NewCounter(MetaConnFailures), PGWireCancelTotalCount: metric.NewCounter(MetaPGWireCancelTotal), diff --git a/pkg/sql/sqlstats/sslocal/sql_stats_test.go b/pkg/sql/sqlstats/sslocal/sql_stats_test.go index cafc94cda195..1a77760bfdb7 100644 --- a/pkg/sql/sqlstats/sslocal/sql_stats_test.go +++ b/pkg/sql/sqlstats/sslocal/sql_stats_test.go @@ -529,11 +529,11 @@ func BenchmarkRecordStatement(b *testing.B) { sqlstats.MaxMemSQLStatsTxnFingerprints, metric.NewGauge(sql.MetaReportedSQLStatsMemCurBytes), /* curMemoryBytesCount */ metric.NewHistogram(metric.HistogramOptions{ - Metadata: sql.MetaReportedSQLStatsMemMaxBytes, - Duration: 10 * time.Second, - MaxVal: 19 * 1000, - SigFigs: 3, - Buckets: metric.MemoryUsage64MBBuckets, + Metadata: sql.MetaReportedSQLStatsMemMaxBytes, + Duration: 10 * time.Second, + MaxVal: 19 * 1000, + SigFigs: 3, + BucketConfig: metric.StaticBucketConfigs[metric.MemoryUsage64MBBuckets], }), /* maxMemoryBytesHist */ insightsProvider.Writer, monitor, diff --git a/pkg/sql/ttl/ttljob/ttljob_metrics.go b/pkg/sql/ttl/ttljob/ttljob_metrics.go index 7670f44a28f6..9c8a787996dc 100644 --- a/pkg/sql/ttl/ttljob/ttljob_metrics.go +++ b/pkg/sql/ttl/ttljob/ttljob_metrics.go @@ -107,10 +107,10 @@ func makeRowLevelTTLAggMetrics(histogramWindowInterval time.Duration) metric.Str Unit: metric.Unit_NANOSECONDS, MetricType: io_prometheus_client.MetricType_HISTOGRAM, }, - MaxVal: time.Hour.Nanoseconds(), - SigFigs: sigFigs, - Duration: histogramWindowInterval, - Buckets: metric.LongRunning60mLatencyBuckets, + MaxVal: time.Hour.Nanoseconds(), + SigFigs: sigFigs, + Duration: histogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.LongRunning60mLatencyBuckets], }), SelectDuration: b.Histogram(metric.HistogramOptions{ Metadata: metric.Metadata{ @@ -120,10 +120,10 @@ func makeRowLevelTTLAggMetrics(histogramWindowInterval time.Duration) metric.Str Unit: metric.Unit_NANOSECONDS, MetricType: io_prometheus_client.MetricType_HISTOGRAM, }, - MaxVal: time.Minute.Nanoseconds(), - SigFigs: sigFigs, - Duration: histogramWindowInterval, - Buckets: metric.BatchProcessLatencyBuckets, + MaxVal: time.Minute.Nanoseconds(), + SigFigs: sigFigs, + Duration: histogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), DeleteDuration: b.Histogram(metric.HistogramOptions{ Metadata: metric.Metadata{ @@ -133,10 +133,10 @@ func makeRowLevelTTLAggMetrics(histogramWindowInterval time.Duration) metric.Str Unit: metric.Unit_NANOSECONDS, MetricType: io_prometheus_client.MetricType_HISTOGRAM, }, - MaxVal: time.Minute.Nanoseconds(), - SigFigs: sigFigs, - Duration: histogramWindowInterval, - Buckets: metric.BatchProcessLatencyBuckets, + MaxVal: time.Minute.Nanoseconds(), + SigFigs: sigFigs, + Duration: histogramWindowInterval, + BucketConfig: metric.StaticBucketConfigs[metric.BatchProcessLatencyBuckets], }), RowSelections: b.Counter( metric.Metadata{ diff --git a/pkg/util/admission/elastic_cpu_granter.go b/pkg/util/admission/elastic_cpu_granter.go index 02a16af28313..6eb3270658ea 100644 --- a/pkg/util/admission/elastic_cpu_granter.go +++ b/pkg/util/admission/elastic_cpu_granter.go @@ -369,10 +369,10 @@ func makeElasticCPUGranterMetrics() *elasticCPUGranterMetrics { AvailableNanos: metric.NewGauge(elasticCPUAvailableNanos), NanosExhaustedDuration: metric.NewGauge(elasticCPUNanosExhaustedDuration), OverLimitDuration: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePrometheus, - Metadata: elasticCPUOverLimitDurations, - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePrometheus, + Metadata: elasticCPUOverLimitDurations, + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), Utilization: metric.NewGaugeFloat64(elasticCPUGranterUtilization), UtilizationLimit: metric.NewGaugeFloat64(elasticCPUGranterUtilizationLimit), diff --git a/pkg/util/admission/work_queue.go b/pkg/util/admission/work_queue.go index 2284e5472fd1..1acf6f56e06e 100644 --- a/pkg/util/admission/work_queue.go +++ b/pkg/util/admission/work_queue.go @@ -1784,10 +1784,10 @@ func makeWorkQueueMetricsSingle(name string) workQueueMetricsSingle { Admitted: metric.NewCounter(addName(name, admittedMeta)), Errored: metric.NewCounter(addName(name, erroredMeta)), WaitDurations: metric.NewHistogram(metric.HistogramOptions{ - Mode: metric.HistogramModePreferHdrLatency, - Metadata: addName(name, waitDurationsMeta), - Duration: base.DefaultHistogramWindowInterval(), - Buckets: metric.IOLatencyBuckets, + Mode: metric.HistogramModePreferHdrLatency, + Metadata: addName(name, waitDurationsMeta), + Duration: base.DefaultHistogramWindowInterval(), + BucketConfig: metric.StaticBucketConfigs[metric.IOLatencyBuckets], }), WaitQueueLength: metric.NewGauge(addName(name, waitQueueLengthMeta)), } diff --git a/pkg/util/metric/aggmetric/agg_metric_test.go b/pkg/util/metric/aggmetric/agg_metric_test.go index 73652961ddea..6ba38ea85230 100644 --- a/pkg/util/metric/aggmetric/agg_metric_test.go +++ b/pkg/util/metric/aggmetric/agg_metric_test.go @@ -72,10 +72,10 @@ func TestAggMetric(t *testing.T) { Metadata: metric.Metadata{ Name: "histo_gram", }, - Duration: base.DefaultHistogramWindowInterval(), - MaxVal: 100, - SigFigs: 1, - Buckets: metric.Count1KBuckets, + Duration: base.DefaultHistogramWindowInterval(), + MaxVal: 100, + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.Count1KBuckets], }, "tenant_id") r.AddMetric(h) @@ -175,11 +175,11 @@ func TestAggMetricBuilder(t *testing.T) { g := b.Gauge(metric.Metadata{Name: "bar_gauge"}) f := b.GaugeFloat64(metric.Metadata{Name: "baz_gauge"}) h := b.Histogram(metric.HistogramOptions{ - Metadata: metric.Metadata{Name: "histo_gram"}, - Duration: base.DefaultHistogramWindowInterval(), - MaxVal: 100, - SigFigs: 1, - Buckets: metric.Count1KBuckets, + Metadata: metric.Metadata{Name: "histo_gram"}, + Duration: base.DefaultHistogramWindowInterval(), + MaxVal: 100, + SigFigs: 1, + BucketConfig: metric.StaticBucketConfigs[metric.Count1KBuckets], }) for i := 5; i < 10; i++ { diff --git a/pkg/util/metric/histogram_buckets.go b/pkg/util/metric/histogram_buckets.go index dcf46deb72b5..55f9ed55ff9c 100644 --- a/pkg/util/metric/histogram_buckets.go +++ b/pkg/util/metric/histogram_buckets.go @@ -10,113 +10,132 @@ package metric -import ( - "github.com/cockroachdb/cockroach/pkg/util/envutil" - "github.com/prometheus/client_golang/prometheus" -) +import "github.com/prometheus/client_golang/prometheus" +// staticBucketConfig describes the buckets we want to generate for a specific +// category of metrics. type staticBucketConfig struct { + category string min float64 max float64 count int - histType histType + units unitType distribution distribution } +// distribution describes the population distribution that best describes the +// metric for which we record histogram data type distribution int -// Distributions const ( Uniform distribution = iota Exponential - // Normal - // LogNormal + // TODO(ericharmeling): add more distributions ) -type histType int +// unitType describes the unit type of the metric for which we record +// histogram data +type unitType int const ( - LATENCY histType = iota + LATENCY unitType = iota SIZE COUNT ) -// precisionTestEnabledEnv enables precision testing buckets for histograms. -const precisionTestEnabledEnv = "COCKROACH_HISTOGRAM_PRECISION_TESTING" -const precisionTestBucketCount = 200 +// histType describes the category of the metric for which we record +// histogram data +type histType int + +const ( + IOLatencyBuckets histType = iota + BatchProcessLatencyBuckets + LongRunning60mLatencyBuckets + DataSize16MBBuckets + MemoryUsage64MBBuckets + ReplicaCPUTimeBuckets + ReplicaBatchRequestCountBuckets + Count1KBuckets + Percent100Buckets +) -var staticBucketConfigs = map[string]staticBucketConfig{ - "IOLatencyBuckets": { - min: 10e3, - max: 10e9, +var StaticBucketConfigs = map[histType]staticBucketConfig{ + IOLatencyBuckets: { + category: "IOLatencyBuckets", + min: 10e3, // 10µs + max: 10e9, // 10s count: 60, - histType: LATENCY, + units: LATENCY, distribution: Exponential, }, - "BatchProcessLatencyBuckets": { - min: 500e6, - max: 300e9, + BatchProcessLatencyBuckets: { + category: "BatchProcessLatencyBuckets", + min: 500e6, // 500ms + max: 300e9, // 5m count: 60, - histType: LATENCY, + units: LATENCY, distribution: Exponential, }, - "LongRunning60mLatencyBuckets": { - min: 500e6, - max: 3600e9, + LongRunning60mLatencyBuckets: { + category: "LongRunning60mLatencyBuckets", + min: 500e6, // 500ms + max: 3600e9, // 1h count: 60, - histType: LATENCY, + units: LATENCY, distribution: Exponential, }, - "Count1KBuckets": { - min: 1, - max: 1024, - count: 11, - histType: COUNT, - distribution: Exponential, - }, - "Percent100Buckets": { - min: 0, - max: 100, - count: 10, - histType: COUNT, - distribution: Uniform, - }, - "DataSize16MBBuckets": { - min: 1e3, - max: 16384e3, + DataSize16MBBuckets: { + category: "DataSize16MBBuckets", + min: 1e3, // 1kB + max: 16384e3, // 16MB count: 15, - histType: SIZE, + units: SIZE, distribution: Exponential, }, - "MemoryUsage64MBBuckets": { - min: 1, - max: 64e6, + MemoryUsage64MBBuckets: { + category: "MemoryUsage64MBBuckets", + min: 1, // 1B + max: 64e6, // 64MB count: 15, - histType: SIZE, + units: SIZE, distribution: Exponential, }, - "ReplicaCPUTimeBuckets": { - min: 50e4, - max: 5e9, + ReplicaCPUTimeBuckets: { + category: "ReplicaCPUTimeBuckets", + min: 50e4, // 500µs + max: 5e9, // 5s count: 20, - histType: LATENCY, + units: LATENCY, distribution: Exponential, }, - "ReplicaBatchRequestCountBuckets": { + ReplicaBatchRequestCountBuckets: { + category: "ReplicaBatchRequestCountBuckets", min: 1, max: 16e3, count: 20, - histType: COUNT, + units: COUNT, + distribution: Exponential, + }, + Count1KBuckets: { + category: "Count1KBuckets", + min: 1, + max: 1024, + count: 11, + units: COUNT, distribution: Exponential, }, + Percent100Buckets: { + category: "Percent100Buckets", + min: 0, + max: 100, + count: 10, + units: COUNT, + distribution: Uniform, + }, } -func getBuckets(config staticBucketConfig) []float64 { +func (config staticBucketConfig) GetBucketsFromBucketConfig() []float64 { var buckets []float64 - if envutil.EnvOrDefaultBool(precisionTestEnabledEnv, false) { - config.distribution = Uniform - config.count = precisionTestBucketCount - } if config.distribution == Uniform { width := (config.max - config.min) / float64(config.count) buckets = prometheus.LinearBuckets(config.min, width, config.count) @@ -126,13 +145,3 @@ func getBuckets(config staticBucketConfig) []float64 { } return buckets } - -var IOLatencyBuckets = getBuckets(staticBucketConfigs["IOLatencyBuckets"]) -var BatchProcessLatencyBuckets = getBuckets(staticBucketConfigs["BatchProcessLatencyBuckets"]) -var LongRunning60mLatencyBuckets = getBuckets(staticBucketConfigs["LongRunning60mLatencyBuckets"]) -var Count1KBuckets = getBuckets(staticBucketConfigs["Count1KBuckets"]) -var Percent100Buckets = getBuckets(staticBucketConfigs["Percent100Buckets"]) -var DataSize16MBBuckets = getBuckets(staticBucketConfigs["DataSize16MBBuckets"]) -var MemoryUsage64MBBuckets = getBuckets(staticBucketConfigs["MemoryUsage64MBBuckets"]) -var ReplicaCPUTimeBuckets = getBuckets(staticBucketConfigs["ReplicaCPUTimeBuckets"]) -var ReplicaBatchRequestCountBuckets = getBuckets(staticBucketConfigs["ReplicaBatchRequestCountBuckets"]) diff --git a/pkg/util/metric/histogram_buckets_test.go b/pkg/util/metric/histogram_buckets_test.go index cc6d7aebfdce..65dd2f3be734 100644 --- a/pkg/util/metric/histogram_buckets_test.go +++ b/pkg/util/metric/histogram_buckets_test.go @@ -35,11 +35,11 @@ func TestHistogramBuckets(t *testing.T) { return buf.String() } - for category, config := range staticBucketConfigs { - exp := getBuckets(config) - buf := verifyAndPrint(t, exp, category) + for _, config := range StaticBucketConfigs { + exp := config.GetBucketsFromBucketConfig() + buf := verifyAndPrint(t, exp, config.category) - echotest.Require(t, buf, datapathutils.TestDataPath(t, category)) + echotest.Require(t, buf, datapathutils.TestDataPath(t, config.category)) } } diff --git a/pkg/util/metric/metric.go b/pkg/util/metric/metric.go index e3508c2ebdb3..9c62c0cf641f 100644 --- a/pkg/util/metric/metric.go +++ b/pkg/util/metric/metric.go @@ -246,6 +246,9 @@ type HistogramOptions struct { // Buckets are only relevant to Prometheus histograms, and represent // the pre-defined histogram bucket boundaries to be used. Buckets []float64 + // BucketConfig is only relevant to Prometheus histograms, and represents + // the pre-defined histogram bucket configuration used to generate buckets. + BucketConfig staticBucketConfig // Mode defines the type of histogram to be used. See individual // comments on each HistogramMode value for details. Mode HistogramMode @@ -259,16 +262,24 @@ func NewHistogram(opt HistogramOptions) IHistogram { return NewHdrHistogram(opt.Metadata, opt.Duration, opt.MaxVal, opt.SigFigs) } } else { - return newHistogram(opt.Metadata, opt.Duration, opt.Buckets) + return newHistogram(opt.Metadata, opt.Duration, opt.Buckets, + opt.BucketConfig) } } // NewHistogram is a prometheus-backed histogram. Depending on the value of // opts.Buckets, this is suitable for recording any kind of quantity. Common // sensible choices are {IO,Network}LatencyBuckets. -func newHistogram(meta Metadata, duration time.Duration, buckets []float64) *Histogram { +func newHistogram( + meta Metadata, duration time.Duration, buckets []float64, bucketConfig staticBucketConfig, +) *Histogram { // TODO(obs-inf): prometheus supports labeled histograms but they require more // plumbing and don't fit into the PrometheusObservable interface any more. + + // If no buckets are provided, generate buckets from bucket configuration + if buckets == nil && bucketConfig.count != 0 { + buckets = bucketConfig.GetBucketsFromBucketConfig() + } opts := prometheus.HistogramOpts{ Buckets: buckets, } diff --git a/pkg/util/metric/metric_test.go b/pkg/util/metric/metric_test.go index a0f9cde324f3..a41e0b4ac9cd 100644 --- a/pkg/util/metric/metric_test.go +++ b/pkg/util/metric/metric_test.go @@ -276,7 +276,6 @@ func TestNewHistogramRotate(t *testing.T) { Mode: HistogramModePrometheus, Metadata: emptyMetadata, Duration: 10 * time.Second, - Buckets: nil, }) for i := 0; i < 4; i++ { // Windowed histogram is initially empty. @@ -306,10 +305,10 @@ func TestHistogramWindowed(t *testing.T) { duration := 10 * time.Second h := NewHistogram(HistogramOptions{ - Mode: HistogramModePrometheus, - Metadata: Metadata{}, - Duration: duration, - Buckets: IOLatencyBuckets, + Mode: HistogramModePrometheus, + Metadata: Metadata{}, + Duration: duration, + BucketConfig: StaticBucketConfigs[IOLatencyBuckets], }) measurements := []int64{200000000, 0, 4000000, 5000000, 10000000, 20000000, @@ -327,6 +326,8 @@ func TestHistogramWindowed(t *testing.T) { // greater than each measurement. count := 0 j := 0 + IOLatencyBuckets := StaticBucketConfigs[IOLatencyBuckets]. + GetBucketsFromBucketConfig() var expQuantileValues []float64 for i := range IOLatencyBuckets { if j < len(sortedMeasurements) && IOLatencyBuckets[i] > float64( @@ -400,7 +401,8 @@ func TestHistogramWindowed(t *testing.T) { func TestMergeWindowedHistogram(t *testing.T) { measurements := []int64{4000000, 90000000} opts := prometheus.HistogramOpts{ - Buckets: IOLatencyBuckets, + Buckets: StaticBucketConfigs[IOLatencyBuckets]. + GetBucketsFromBucketConfig(), } prevWindow := prometheus.NewHistogram(opts) diff --git a/pkg/util/metric/registry_test.go b/pkg/util/metric/registry_test.go index 48f4aba216bd..ca3ad776ac48 100644 --- a/pkg/util/metric/registry_test.go +++ b/pkg/util/metric/registry_test.go @@ -77,10 +77,10 @@ func TestRegistry(t *testing.T) { r.AddMetric(topCounter) r.AddMetric(NewHistogram(HistogramOptions{ - Mode: HistogramModePrometheus, - Metadata: Metadata{Name: "top.histogram"}, - Duration: time.Minute, - Buckets: Count1KBuckets, + Mode: HistogramModePrometheus, + Metadata: Metadata{Name: "top.histogram"}, + Duration: time.Minute, + BucketConfig: StaticBucketConfigs[Count1KBuckets], })) r.AddMetric(NewGauge(Metadata{Name: "bottom.gauge"})) @@ -109,10 +109,10 @@ func TestRegistry(t *testing.T) { StructGauge64: NewGaugeFloat64(Metadata{Name: "struct.gauge64"}), StructCounter: NewCounter(Metadata{Name: "struct.counter"}), StructHistogram: NewHistogram(HistogramOptions{ - Mode: HistogramModePrometheus, - Metadata: Metadata{Name: "struct.histogram"}, - Duration: time.Minute, - Buckets: Count1KBuckets, + Mode: HistogramModePrometheus, + Metadata: Metadata{Name: "struct.histogram"}, + Duration: time.Minute, + BucketConfig: StaticBucketConfigs[Count1KBuckets], }), NestedStructGauge: NestedStruct{ NestedStructGauge: NewGauge(Metadata{Name: "nested.struct.gauge"}), @@ -133,10 +133,10 @@ func TestRegistry(t *testing.T) { privateStructGauge64: NewGaugeFloat64(Metadata{Name: "private.struct.gauge64"}), privateStructCounter: NewCounter(Metadata{Name: "private.struct.counter"}), privateStructHistogram: NewHistogram(HistogramOptions{ - Mode: HistogramModePrometheus, - Metadata: Metadata{Name: "private.struct.histogram"}, - Duration: time.Minute, - Buckets: Count1KBuckets, + Mode: HistogramModePrometheus, + Metadata: Metadata{Name: "private.struct.histogram"}, + Duration: time.Minute, + BucketConfig: StaticBucketConfigs[Count1KBuckets], }), privateNestedStructGauge: NestedStruct{ NestedStructGauge: NewGauge(Metadata{Name: "private.nested.struct.gauge"}),