Queries dashboard: allow using cortex_request_duration_seconds native…

… histogram Signed-off-by: Yuri Nikolic <[email protected]>
grafana · Jul 22, 2024 · c1595cd · c1595cd
1 parent af7fe67
commit c1595cd
Show file tree

Hide file tree

Showing 6 changed files with 102 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -68,6 +68,7 @@
   * Writes dashboard: `cortex_request_duration_seconds` metric. #8757 #8791
   * Reads dashboard: `cortex_request_duration_seconds` metric. #8752
   * Rollout progress dashboard. #8779
+  * Queries dashboard: `cortex_request_duration_seconds` metric. #8800
 * [ENHANCEMENT] Alerts: `MimirRunningIngesterReceiveDelayTooHigh` alert has been tuned to be more reactive to high receive delay. #8538
 * [ENHANCEMENT] Dashboards: improve end-to-end latency and strong read consistency panels when experimental ingest storage is enabled. #8543
 * [ENHANCEMENT] Dashboards: Add panels for monitoring ingester autoscaling when not using ingest-storage. These panels are disabled by default, but can be enabled using the `autoscaling.ingester.enabled: true` config option. #8484

diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -34,6 +34,7 @@ Entries should include a reference to the Pull Request that introduced the chang
   * Writes dashboard: `cortex_request_duration_seconds` metric. #8757
   * Reads dashboard: `cortex_request_duration_seconds` metric. #8752
   * Rollout progress dashboard. #8779
+  * Queries dashboard: `cortex_request_duration_seconds` metric. #8800
 * [ENHANCEMENT] Memcached: Update to Memcached 1.6.28 and memcached-exporter 0.14.4. #8557
 * [ENHANCEMENT] Add missing fields in multiple topology spread constraints. #8533
 * [ENHANCEMENT] Add support for setting the image pull secrets, node selectors, tolerations and topology spread constraints for the Grafana Agent pods used for metamonitoring. #8670

diff --git a/...oring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml b/...oring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml
@@ -13583,6 +13583,35 @@ data:
                    "tagsQuery": "",
                    "type": "query",
                    "useTags": false
+                },
+                {
+                   "current": {
+                      "selected": true,
+                      "text": "classic",
+                      "value": "1"
+                   },
+                   "description": "Choose between showing latencies based on low precision classic or high precision native histogram metrics.",
+                   "hide": 0,
+                   "includeAll": false,
+                   "label": "Latency metrics",
+                   "multi": false,
+                   "name": "latency_metrics",
+                   "options": [
+                      {
+                         "selected": false,
+                         "text": "native",
+                         "value": "-1"
+                      },
+                      {
+                         "selected": true,
+                         "text": "classic",
+                         "value": "1"
+                      }
+                   ],
+                   "query": "native : -1,classic : 1",
+                   "skipUrlSync": false,
+                   "type": "custom",
+                   "useTags": false
                 }
              ]
           },

diff --git a/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-queries.json b/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-queries.json
diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-queries.json b/operations/mimir-mixin-compiled/dashboards/mimir-queries.json
diff --git a/operations/mimir-mixin/dashboards/queries.libsonnet b/operations/mimir-mixin/dashboards/queries.libsonnet
@@ -6,6 +6,7 @@ local filename = 'mimir-queries.json';
     assert std.md5(filename) == 'b3abe8d5c040395cc36615cb4334c92d' : 'UID of the dashboard has changed, please update references to dashboard.';
     ($.dashboard('Queries') + { uid: std.md5(filename) })
     .addClusterSelectorTemplates()
+    .addShowNativeLatencyVariable()
     .addRow(
       $.row('Query-frontend')
       .addPanel(
@@ -235,23 +236,27 @@ local filename = 'mimir-queries.json';
           |||
         ) +
         $.queryPanel(
-          [
+          local ncSumRate = utils.ncHistogramSumBy(utils.ncHistogramCountRate($.queries.ingester.requestsPerSecondMetric, $.queries.ingester.readRequestsPerSecondSelector));
+          local scSuccessful =
             |||
               (
                 sum(rate(cortex_ingest_storage_strong_consistency_requests_total{%s}[$__rate_interval]))
                 -
                 sum(rate(cortex_ingest_storage_strong_consistency_failures_total{%s}[$__rate_interval]))
               )
-              /
-              sum(rate(cortex_request_duration_seconds_count{%s,route=~"%s"}[$__rate_interval]))
-            ||| % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester), $._config.ingester_read_path_routes_regex],
+            ||| % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester)];
+          local scFailed =
             |||
               sum(rate(cortex_ingest_storage_strong_consistency_failures_total{%s}[$__rate_interval]))
-              /
-              sum(rate(cortex_request_duration_seconds_count{%s,route=~"%s"}[$__rate_interval]))
-            ||| % [$.jobMatcher($._config.job_names.ingester), $.jobMatcher($._config.job_names.ingester), $._config.ingester_read_path_routes_regex],
+            ||| % [$.jobMatcher($._config.job_names.ingester)];
+          local scRate(sc, rate) = std.join(' / ', [sc, rate]);
+          [
+            scRate(scSuccessful, utils.showClassicHistogramQuery(ncSumRate)),
+            scRate(scSuccessful, utils.showNativeHistogramQuery(ncSumRate)),
+            scRate(scFailed, utils.showClassicHistogramQuery(ncSumRate)),
+            scRate(scFailed, utils.showNativeHistogramQuery(ncSumRate)),
           ],
-          ['successful', 'failed'],
+          ['successful', 'successful', 'failed', 'failed'],
         )
         + $.aliasColors({ failed: $._colors.failed, successful: $._colors.success })
         + { fieldConfig+: { defaults+: { unit: 'percentunit', min: 0, max: 1 } } }