diff --git a/CHANGELOG.md b/CHANGELOG.md index 959685b9d96..5037af7b53a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,7 @@ * [ENHANCEMENT] Dashboards: Render graph panels at full resolution as opposed to at half resolution. #7027 * [ENHANCEMENT] Dashboards: show query-scheduler queue length on "Reads" and "Remote Ruler Reads" dashboards. #7088 * [ENHANCEMENT] Dashboards: Add estimated number of compaction jobs to "Compactor", "Tenants" and "Top tenants" dashboards. #7449 #7481 +* [ENHANCEMENT] Recording rules: add native histogram recording rules to `cortex_request_duration_seconds`. #7528 * [BUGFIX] Dashboards: drop `step` parameter from targets as it is not supported. #7157 * [BUGFIX] Recording rules: drop rules for metrics removed in 2.0: `cortex_memcache_request_duration_seconds` and `cortex_cache_request_duration_seconds`. #7514 diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md index 131ca927937..0f0593ebdae 100644 --- a/operations/helm/charts/mimir-distributed/CHANGELOG.md +++ b/operations/helm/charts/mimir-distributed/CHANGELOG.md @@ -55,6 +55,7 @@ Entries should include a reference to the Pull Request that introduced the chang * [ENHANCEMENT] nginx, Gateway: set `proxy_http_version: 1.1` to proxy to HTTP 1.1. #5040 * [ENHANCEMENT] Gateway: make Ingress/Route host templateable. #7218 * [ENHANCEMENT] Make the PSP template configurable via `rbac.podSecurityPolicy`. #7190 +* [ENHANCEMENT] Recording rules: add native histogram recording rules to `cortex_request_duration_seconds`. #7528 * [BUGFIX] Metamonitoring: update dashboards to drop unsupported `step` parameter in targets. #7157 * [BUGFIX] Recording rules: drop rules for metrics removed in 2.0: `cortex_memcache_request_duration_seconds` and `cortex_cache_request_duration_seconds`. #7514 diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml index 021d4f20788..304e695fd6b 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml @@ -28,6 +28,8 @@ spec: record: cluster_job:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job) record: cluster_job:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job) + record: cluster_job:cortex_request_duration_seconds:sum_rate - name: mimir_api_2 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -46,6 +48,8 @@ spec: record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route) record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route) + record: cluster_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_api_3 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -67,6 +71,9 @@ spec: - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace, job, route) record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job, + route) + record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_querier_api rules: - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) diff --git a/operations/mimir-mixin-compiled-baremetal/rules.yaml b/operations/mimir-mixin-compiled-baremetal/rules.yaml index 8a72b11dd4a..a88ae0d4f9a 100644 --- a/operations/mimir-mixin-compiled-baremetal/rules.yaml +++ b/operations/mimir-mixin-compiled-baremetal/rules.yaml @@ -16,6 +16,8 @@ groups: record: cluster_job:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job) record: cluster_job:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job) + record: cluster_job:cortex_request_duration_seconds:sum_rate - name: mimir_api_2 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -34,6 +36,8 @@ groups: record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route) record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route) + record: cluster_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_api_3 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -55,6 +59,9 @@ groups: - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace, job, route) record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job, + route) + record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_querier_api rules: - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) diff --git a/operations/mimir-mixin-compiled/rules.yaml b/operations/mimir-mixin-compiled/rules.yaml index 91ed5d591a1..bd28aa9ec03 100644 --- a/operations/mimir-mixin-compiled/rules.yaml +++ b/operations/mimir-mixin-compiled/rules.yaml @@ -16,6 +16,8 @@ groups: record: cluster_job:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job) record: cluster_job:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job) + record: cluster_job:cortex_request_duration_seconds:sum_rate - name: mimir_api_2 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -34,6 +36,8 @@ groups: record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route) record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route) + record: cluster_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_api_3 rules: - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m])) @@ -55,6 +59,9 @@ groups: - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace, job, route) record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate + - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job, + route) + record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate - name: mimir_querier_api rules: - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) diff --git a/operations/mimir-mixin/jsonnetfile.lock.json b/operations/mimir-mixin/jsonnetfile.lock.json index b67774b2744..2dde18000fd 100644 --- a/operations/mimir-mixin/jsonnetfile.lock.json +++ b/operations/mimir-mixin/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "grafana-builder" } }, - "version": "3d58bd591c278f3f342bc1e25399806c49ace104", + "version": "0098700428a0a4ee7d884d332d137caff5c52497", "sum": "B49EzIY2WZsFxNMJcgRxE/gcZ9ltnS8pkOOV6Q5qioc=" }, { @@ -18,8 +18,8 @@ "subdir": "mixin-utils" } }, - "version": "3d58bd591c278f3f342bc1e25399806c49ace104", - "sum": "vyT1akj0RbnIeb0L3cJ/HzLiOEm5lskwl/Xr34eHOZQ=" + "version": "0098700428a0a4ee7d884d332d137caff5c52497", + "sum": "EWPd0a5uU5x1vTuyyMbH+d41wrgem7v21c2p4jekkbA=" } ], "legacyImports": false diff --git a/operations/mimir-mixin/recording_rules.libsonnet b/operations/mimir-mixin/recording_rules.libsonnet index 73099805781..567bae04989 100644 --- a/operations/mimir-mixin/recording_rules.libsonnet +++ b/operations/mimir-mixin/recording_rules.libsonnet @@ -12,17 +12,17 @@ local utils = import 'mixin-utils/utils.libsonnet'; { name: 'mimir_api_1', rules: - utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job'], $._config.recording_rules_range_interval), + utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job'], $._config.recording_rules_range_interval, record_native=true), }, { name: 'mimir_api_2', rules: - utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job', 'route'], $._config.recording_rules_range_interval), + utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job', 'route'], $._config.recording_rules_range_interval, record_native=true), }, { name: 'mimir_api_3', rules: - utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels + ['route'], $._config.recording_rules_range_interval), + utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels + ['route'], $._config.recording_rules_range_interval, record_native=true), }, { name: 'mimir_querier_api',