diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f8b7abab9..7534e465d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * [ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. #6296 * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276 * [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 +* [ENHANCEMENT] Distributor: Expose `cortex_label_size_bytes` native histogram metric. #6372 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 * [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index be94cfa2f1..2cb8999a0a 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -78,6 +78,7 @@ type ValidateMetrics struct { DiscardedExemplars *prometheus.CounterVec DiscardedMetadata *prometheus.CounterVec HistogramSamplesReducedResolution *prometheus.CounterVec + LabelSizeBytes *prometheus.HistogramVec } func registerCollector(r prometheus.Registerer, c prometheus.Collector) { @@ -120,11 +121,21 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { []string{"user"}, ) registerCollector(r, histogramSamplesReducedResolution) + labelSizeBytes := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_label_size_bytes", + Help: "The combined size in bytes of all labels and label values for a time series.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"user"}) + registerCollector(r, labelSizeBytes) + m := &ValidateMetrics{ DiscardedSamples: discardedSamples, DiscardedExemplars: discardedExemplars, DiscardedMetadata: discardedMetadata, HistogramSamplesReducedResolution: histogramSamplesReducedResolution, + LabelSizeBytes: labelSizeBytes, } return m @@ -236,6 +247,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str lastLabelName = l.Name labelsSizeBytes += l.Size() } + validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc() return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 7d7cf45930..6b6fb00bbf 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -119,6 +119,14 @@ func TestValidateLabels(t *testing.T) { cortex_discarded_samples_total{reason="random reason",user="different user"} 1 `), "cortex_discarded_samples_total")) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_label_size_bytes The combined size in bytes of all labels and label values for a time series. + # TYPE cortex_label_size_bytes histogram + cortex_label_size_bytes_bucket{user="testUser",le="+Inf"} 3 + cortex_label_size_bytes_sum{user="testUser"} 148 + cortex_label_size_bytes_count{user="testUser"} 3 + `), "cortex_label_size_bytes")) + DeletePerUserValidationMetrics(validateMetrics, userID, util_log.Logger) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`