From 7a7e3f98e75a5cedece0180acaa61783dca2f901 Mon Sep 17 00:00:00 2001 From: Radu Berinde Date: Fri, 25 Oct 2024 18:11:19 -0700 Subject: [PATCH] kvserver: preinitialize some Pebble category stats This change fixes non-determinism when generating `metrics.html` due to the Pebble category stats. Informs #133507 Unblocks #133409 Epic: none Release note: None --- docs/generated/metrics/metrics.html | 18 ++++++++++++++++++ pkg/kv/kvserver/metrics.go | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/generated/metrics/metrics.html b/docs/generated/metrics/metrics.html index 261517ad10b9..fe659599b919 100644 --- a/docs/generated/metrics/metrics.html +++ b/docs/generated/metrics/metrics.html @@ -722,6 +722,24 @@ STORAGEstorage.iterator.block-load.bytesBytes loaded by storage engine iterators (possibly cached). See storage.AggregatedIteratorStats for details.BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE STORAGEstorage.iterator.block-load.cached-bytesBytes loaded by storage engine iterators from the block cache. See storage.AggregatedIteratorStats for details.BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE STORAGEstorage.iterator.block-load.read-durationCumulative time storage engine iterators spent loading blocks from durable storage. See storage.AggregatedIteratorStats for details.NanosecondsCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-batch-eval.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-batch-eval.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-batch-eval.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-crdb-unknown.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-crdb-unknown.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-crdb-unknown.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-mvcc-gc.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-mvcc-gc.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-mvcc-gc.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-rangefeed.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-rangefeed.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-rangefeed.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-replication.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-replication.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-replication.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-scan-regular.block-load.bytesBytes loaded by storage sstable iterators (possibly cached).BytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-scan-regular.block-load.cached-bytesBytes loaded by storage sstable iterators from the block cacheBytesCOUNTERBYTESAVGNON_NEGATIVE_DERIVATIVE +STORAGEstorage.iterator.category-scan-regular.block-load.latency-sumCumulative latency for loading bytes not in the block cache, by storage sstable iteratorsLatencyCOUNTERNANOSECONDSAVGNON_NEGATIVE_DERIVATIVE STORAGEstorage.iterator.external.seeksCumulative count of seeks performed on storage engine iterators. See storage.AggregatedIteratorStats for details.Iterator OpsCOUNTERCOUNTAVGNON_NEGATIVE_DERIVATIVE STORAGEstorage.iterator.external.stepsCumulative count of steps performed on storage engine iterators. See storage.AggregatedIteratorStats for details.Iterator OpsCOUNTERCOUNTAVGNON_NEGATIVE_DERIVATIVE STORAGEstorage.iterator.internal.seeksCumulative count of seeks performed internally within storage engine iterators.

A value high relative to 'storage.iterator.external.seeks'
is a good indication that there's an accumulation of garbage
internally within the storage engine.

See storage.AggregatedIteratorStats for details.Iterator OpsCOUNTERCOUNTAVGNON_NEGATIVE_DERIVATIVE diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index 1a576e6707e6..d3977dcdfa32 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -3753,6 +3753,17 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { storeRegistry.AddMetricStruct(sm) storeRegistry.AddMetricStruct(sm.LoadSplitterMetrics) + + // Pre-initialize some category stats, so that metrics generation is + // deterministic. + // TODO(radu): #133507 tracks fixing this properly. + for _, category := range []sstable.Category{ + "batch-eval", "crdb-unknown", "mvcc-gc", "rangefeed", + "replication", "scan-regular"} { + cm := makePebbleCategorizedIterMetrics(category) + sm.categoryIterMetrics.metricsMap.Store(category, cm) + storeRegistry.AddMetricStruct(cm) + } return sm }