Fix promtail requests error alert (#1344)

giantswarm · Sep 3, 2024 · 655403f · 655403f
1 parent 7ef7a32
commit 655403f
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+Fix `PromtailRequestsErrors` alerts as promtail retries after some backoff so actual errors are hidden.
+
 ## [4.13.0] - 2024-09-03
 
 ### Changed

diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/promtail.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/promtail.rules.yml
@@ -36,7 +36,7 @@ spec:
             description: This alert checks if that the amount of failed requests is below 10% for promtail
             opsrecipe: promtail/
           expr: |
-            100 * (sum(rate(promtail_request_duration_seconds_count{status_code!~"2.."}[2m])) by (cluster_id, installation, provider, pipeline, namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[2m])) by (cluster_id, installation, provider, pipeline, namespace, job, route, instance)) > 10
+            100 * (sum(rate(promtail_request_duration_seconds_count{status_code!~"2.."}[5m])) by (cluster_id, installation, provider, pipeline, namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[5m])) by (cluster_id, installation, provider, pipeline, namespace, job, route, instance)) > 10
           for: 15m
           labels:
             area: platform