diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b8b513..4622bf90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- MimirObjectStorageLowRate and LokiObjectStorageLowRate only check management cluster apps +- MimirObjectStorageLowRate and LokiObjectStorageLowRate are less sensitive + ## [4.26.0] - 2024-11-19 ### Changed diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml index 2859e749..fb051b84 100644 --- a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml +++ b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/loki.rules.yml @@ -201,7 +201,7 @@ spec: description: '{{`Loki object storage write rate is down.`}}' opsrecipe: loki/ expr: | - irate(loki_rate_store_stream_rate_bytes_count[5m]) == 0 + rate(loki_rate_store_stream_rate_bytes_count{cluster_type="management_cluster"}[30m]) == 0 # This part will fire the alert when the metric does not exist or ( label_replace( @@ -212,7 +212,7 @@ spec: "(.*)" ) == 1 ) unless on (cluster_id) ( - count(loki_rate_store_stream_rate_bytes_count) by (cluster_id) + count(loki_rate_store_stream_rate_bytes_count{cluster_type="management_cluster"}) by (cluster_id) ) for: 1h labels: diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml index 2cf4addf..d6b8836d 100644 --- a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml +++ b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml @@ -272,7 +272,7 @@ spec: description: '{{`Mimir object storage write rate is down.`}}' opsrecipe: mimir/ expr: | - irate(cortex_bucket_store_sent_chunk_size_bytes_count[5m]) == 0 + rate(cortex_bucket_store_sent_chunk_size_bytes_count{cluster_type="management_cluster"}[30m]) == 0 # This part will fire the alert when the metric does not exist or ( label_replace( @@ -283,7 +283,7 @@ spec: "(.*)" ) == 1 ) unless on (cluster_id) ( - count(cortex_bucket_store_sent_chunk_size_bytes_count) by (cluster_id) + count(cortex_bucket_store_sent_chunk_size_bytes_count{cluster_type="management_cluster"}) by (cluster_id) ) for: 1h labels: diff --git a/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml b/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml index 5f0333ce..19833a41 100644 --- a/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml +++ b/test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml @@ -571,9 +571,9 @@ tests: - interval: 1m input_series: - series: 'cortex_bucket_store_sent_chunk_size_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capa"}' - values: "_x90 1+1x90 90+0x90" + values: "_x90 1+1x90 90+0x200" - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capa", name="myinstall", type="ControlPlaneReady", status="True"}' - values: "1+0x270" + values: "1+0x380" alert_rule_test: - alertname: MimirObjectStorageLowRate eval_time: 40m @@ -608,7 +608,7 @@ tests: - alertname: MimirObjectStorageLowRate eval_time: 200m - alertname: MimirObjectStorageLowRate - eval_time: 250m + eval_time: 300m exp_alerts: - exp_labels: area: platform diff --git a/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml b/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml index 7050bb8d..1314630e 100644 --- a/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml +++ b/test/tests/providers/capi/capz/platform/atlas/alerting-rules/mimir.rules.test.yml @@ -571,9 +571,9 @@ tests: - interval: 1m input_series: - series: 'cortex_bucket_store_sent_chunk_size_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capz"}' - values: "_x90 1+1x90 90+0x90" + values: "_x90 1+1x90 90+0x200" - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="mimir", pipeline="stable", provider="capz", name="myinstall", type="ControlPlaneReady", status="True"}' - values: "1+0x270" + values: "1+0x380" alert_rule_test: - alertname: MimirObjectStorageLowRate eval_time: 40m @@ -608,7 +608,7 @@ tests: - alertname: MimirObjectStorageLowRate eval_time: 200m - alertname: MimirObjectStorageLowRate - eval_time: 250m + eval_time: 300m exp_alerts: - exp_labels: area: platform diff --git a/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml b/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml index 88d28989..7b0995a8 100644 --- a/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml +++ b/test/tests/providers/global/platform/atlas/alerting-rules/loki.rules.test.yml @@ -329,9 +329,9 @@ tests: - interval: 1m input_series: - series: 'loki_rate_store_stream_rate_bytes_count{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="loki", pipeline="stable"}' - values: "_x90 1+1x90 90+0x90" + values: "_x90 1+1x90 90+0x200" - series: 'capi_cluster_status_condition{cluster_id="myinstall", cluster_type="management_cluster", installation="myinstall", namespace="loki", pipeline="stable", name="myinstall", type="ControlPlaneReady", status="True"}' - values: "1+0x270" + values: "1+0x380" alert_rule_test: - alertname: LokiObjectStorageLowRate eval_time: 40m @@ -364,7 +364,7 @@ tests: - alertname: LokiObjectStorageLowRate eval_time: 200m - alertname: LokiObjectStorageLowRate - eval_time: 250m + eval_time: 300m exp_alerts: - exp_labels: area: platform