improve loki and mimir object storage alerts (#1430)

Co-authored-by: Herve Nicol <[email protected]>
giantswarm · Nov 19, 2024 · 2e0e8d2 · 2e0e8d2
1 parent 6b82c35
commit 2e0e8d2
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- MimirObjectStorageLowRate and LokiObjectStorageLowRate only check management cluster apps
+- MimirObjectStorageLowRate and LokiObjectStorageLowRate are less sensitive
+
 ## [4.26.0] - 2024-11-19
 
 ### Changed

diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml
@@ -201,7 +201,7 @@ spec:
         description: '{{`Loki object storage write rate is down.`}}'
         opsrecipe: loki/
       expr: |
-        irate(loki_rate_store_stream_rate_bytes_count[5m]) == 0
+        rate(loki_rate_store_stream_rate_bytes_count{cluster_type="management_cluster"}[30m]) == 0
         # This part will fire the alert when the metric does not exist
         or (
             label_replace(
@@ -212,7 +212,7 @@ spec:
               "(.*)"
             ) == 1
           ) unless on (cluster_id) (
-            count(loki_rate_store_stream_rate_bytes_count) by (cluster_id)
+            count(loki_rate_store_stream_rate_bytes_count{cluster_type="management_cluster"}) by (cluster_id)
           )
       for: 1h
       labels:

diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml
@@ -272,7 +272,7 @@ spec:
         description: '{{`Mimir object storage write rate is down.`}}'
         opsrecipe: mimir/
       expr: |
-        irate(cortex_bucket_store_sent_chunk_size_bytes_count[5m]) == 0
+        rate(cortex_bucket_store_sent_chunk_size_bytes_count{cluster_type="management_cluster"}[30m]) == 0
         # This part will fire the alert when the metric does not exist
         or (
             label_replace(
@@ -283,7 +283,7 @@ spec:
               "(.*)"
             ) == 1
           ) unless on (cluster_id) (
-            count(cortex_bucket_store_sent_chunk_size_bytes_count) by (cluster_id)
+            count(cortex_bucket_store_sent_chunk_size_bytes_count{cluster_type="management_cluster"}) by (cluster_id)
           )
       for: 1h
       labels:

diff --git a/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml b/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml
@@ -571,9 +571,9 @@ tests:
   - interval: 1m
     input_series:
       - series: 'cortex_bucket_store_sent_chunk_size_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capa"}'
-        values: "_x90 1+1x90 90+0x90"
+        values: "_x90 1+1x90 90+0x200"
       - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capa", name="myinstall", type="ControlPlaneReady", status="True"}'
-        values: "1+0x270"
+        values: "1+0x380"
     alert_rule_test:
       - alertname: MimirObjectStorageLowRate
         eval_time: 40m
@@ -608,7 +608,7 @@ tests:
       - alertname: MimirObjectStorageLowRate
         eval_time: 200m
       - alertname: MimirObjectStorageLowRate
-        eval_time: 250m
+        eval_time: 300m
         exp_alerts:
           - exp_labels:
               area: platform

diff --git a/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml b/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml
@@ -571,9 +571,9 @@ tests:
   - interval: 1m
     input_series:
       - series: 'cortex_bucket_store_sent_chunk_size_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capz"}'
-        values: "_x90 1+1x90 90+0x90"
+        values: "_x90 1+1x90 90+0x200"
       - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capz", name="myinstall", type="ControlPlaneReady", status="True"}'
-        values: "1+0x270"
+        values: "1+0x380"
     alert_rule_test:
       - alertname: MimirObjectStorageLowRate
         eval_time: 40m
@@ -608,7 +608,7 @@ tests:
       - alertname: MimirObjectStorageLowRate
         eval_time: 200m
       - alertname: MimirObjectStorageLowRate
-        eval_time: 250m
+        eval_time: 300m
         exp_alerts:
           - exp_labels:
               area: platform

diff --git a/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml b/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml
@@ -329,9 +329,9 @@ tests:
   - interval: 1m
     input_series:
       - series: 'loki_rate_store_stream_rate_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="loki", pipeline="stable"}'
-        values: "_x90 1+1x90 90+0x90"
+        values: "_x90 1+1x90 90+0x200"
       - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="loki", pipeline="stable", name="myinstall", type="ControlPlaneReady", status="True"}'
-        values: "1+0x270"
+        values: "1+0x380"
     alert_rule_test:
       - alertname: LokiObjectStorageLowRate
         eval_time: 40m
@@ -364,7 +364,7 @@ tests:
       - alertname: LokiObjectStorageLowRate
         eval_time: 200m
       - alertname: LokiObjectStorageLowRate
-        eval_time: 250m
+        eval_time: 300m
         exp_alerts:
           - exp_labels:
               area: platform