mixin: native histogram recording rule: cortex_request_duration_seconds

Add native histogram recording rule for cortex_request_duration_seconds. This is one of two main histograms that are included in alerts. The new recording rule is required to be able to drop the classic histograms in the mixin. Signed-off-by: György Krajcsovits <[email protected]>
grafana · Mar 4, 2024 · 85a8ac9 · 85a8ac9
1 parent 0d171f9
commit 85a8ac9
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -126,6 +126,7 @@
 * [ENHANCEMENT] Dashboards: Render graph panels at full resolution as opposed to at half resolution. #7027
 * [ENHANCEMENT] Dashboards: show query-scheduler queue length on "Reads" and "Remote Ruler Reads" dashboards. #7088
 * [ENHANCEMENT] Dashboards: Add estimated number of compaction jobs to "Compactor", "Tenants" and "Top tenants" dashboards. #7449 #7481
+* [ENHANCEMENT] Recording rules: add native histogram recording rules to `cortex_request_duration_seconds`. #7528
 * [BUGFIX] Dashboards: drop `step` parameter from targets as it is not supported. #7157
 * [BUGFIX] Recording rules: drop rules for metrics removed in 2.0: `cortex_memcache_request_duration_seconds` and `cortex_cache_request_duration_seconds`. #7514
 

diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -55,6 +55,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [ENHANCEMENT] nginx, Gateway: set `proxy_http_version: 1.1` to proxy to HTTP 1.1. #5040
 * [ENHANCEMENT] Gateway: make Ingress/Route host templateable. #7218
 * [ENHANCEMENT] Make the PSP template configurable via `rbac.podSecurityPolicy`. #7190
+* [ENHANCEMENT] Recording rules: add native histogram recording rules to `cortex_request_duration_seconds`. #7528
 * [BUGFIX] Metamonitoring: update dashboards to drop unsupported `step` parameter in targets. #7157
 * [BUGFIX] Recording rules: drop rules for metrics removed in 2.0: `cortex_memcache_request_duration_seconds` and `cortex_cache_request_duration_seconds`. #7514
 

diff --git a/...nitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml b/...nitoring-values-generated/mimir-distributed/templates/metamonitoring/recording-rules.yaml
@@ -28,6 +28,8 @@ spec:
       record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
     - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
       record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+    - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job)
+      record: cluster_job:cortex_request_duration_seconds:sum_rate
   - name: mimir_api_2
     rules:
     - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -46,6 +48,8 @@ spec:
       record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
     - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
       record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
+    - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route)
+      record: cluster_job_route:cortex_request_duration_seconds:sum_rate
   - name: mimir_api_3
     rules:
     - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -67,6 +71,9 @@ spec:
     - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
         job, route)
       record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
+    - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job,
+        route)
+      record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate
   - name: mimir_querier_api
     rules:
     - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))

diff --git a/operations/mimir-mixin-compiled-baremetal/rules.yaml b/operations/mimir-mixin-compiled-baremetal/rules.yaml
@@ -16,6 +16,8 @@ groups:
     record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
     record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job)
+    record: cluster_job:cortex_request_duration_seconds:sum_rate
 - name: mimir_api_2
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -34,6 +36,8 @@ groups:
     record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
     record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route)
+    record: cluster_job_route:cortex_request_duration_seconds:sum_rate
 - name: mimir_api_3
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -55,6 +59,9 @@ groups:
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
       job, route)
     record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job,
+      route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate
 - name: mimir_querier_api
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))

diff --git a/operations/mimir-mixin-compiled/rules.yaml b/operations/mimir-mixin-compiled/rules.yaml
@@ -16,6 +16,8 @@ groups:
     record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
     record: cluster_job:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job)
+    record: cluster_job:cortex_request_duration_seconds:sum_rate
 - name: mimir_api_2
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -34,6 +36,8 @@ groups:
     record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
     record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, job, route)
+    record: cluster_job_route:cortex_request_duration_seconds:sum_rate
 - name: mimir_api_3
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
@@ -55,6 +59,9 @@ groups:
   - expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
       job, route)
     record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
+  - expr: sum(rate(cortex_request_duration_seconds[1m])) by (cluster, namespace, job,
+      route)
+    record: cluster_namespace_job_route:cortex_request_duration_seconds:sum_rate
 - name: mimir_querier_api
   rules:
   - expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))

diff --git a/operations/mimir-mixin/jsonnetfile.lock.json b/operations/mimir-mixin/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "3d58bd591c278f3f342bc1e25399806c49ace104",
+      "version": "0098700428a0a4ee7d884d332d137caff5c52497",
       "sum": "B49EzIY2WZsFxNMJcgRxE/gcZ9ltnS8pkOOV6Q5qioc="
     },
     {
@@ -18,8 +18,8 @@
           "subdir": "mixin-utils"
         }
       },
-      "version": "3d58bd591c278f3f342bc1e25399806c49ace104",
-      "sum": "vyT1akj0RbnIeb0L3cJ/HzLiOEm5lskwl/Xr34eHOZQ="
+      "version": "0098700428a0a4ee7d884d332d137caff5c52497",
+      "sum": "EWPd0a5uU5x1vTuyyMbH+d41wrgem7v21c2p4jekkbA="
     }
   ],
   "legacyImports": false

diff --git a/operations/mimir-mixin/recording_rules.libsonnet b/operations/mimir-mixin/recording_rules.libsonnet
@@ -12,17 +12,17 @@ local utils = import 'mixin-utils/utils.libsonnet';
       {
         name: 'mimir_api_1',
         rules:
-          utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job'], $._config.recording_rules_range_interval),
+          utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job'], $._config.recording_rules_range_interval, record_native=true),
       },
       {
         name: 'mimir_api_2',
         rules:
-          utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job', 'route'], $._config.recording_rules_range_interval),
+          utils.histogramRules('cortex_request_duration_seconds', [$._config.per_cluster_label, 'job', 'route'], $._config.recording_rules_range_interval, record_native=true),
       },
       {
         name: 'mimir_api_3',
         rules:
-          utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels + ['route'], $._config.recording_rules_range_interval),
+          utils.histogramRules('cortex_request_duration_seconds', $._config.job_labels + ['route'], $._config.recording_rules_range_interval, record_native=true),
       },
       {
         name: 'mimir_querier_api',