add team in SLO alert (#971)

Signed-off-by: QuentinBisson <[email protected]>
giantswarm · Nov 27, 2023 · 13553dc · 13553dc
1 parent 2ae6cf7
commit 13553dc
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Relabel team in `ServiceLevelBurnRateTooHigh` alert.
+
 ## [2.143.2] - 2023-11-22
 
 ### Changed

diff --git a/helm/prometheus-rules/templates/alerting-rules/service-level.rules.yml b/helm/prometheus-rules/templates/alerting-rules/service-level.rules.yml
@@ -16,16 +16,30 @@ spec:
         opsrecipe: service-level-burn-rate-too-high/
         dashboard: https://giantswarm.grafana.net/d/service-level/service-level?orgId=1
       expr: |
-        (
-          slo_errors_per_request:ratio_rate1h{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"} > on (service) group_left slo_threshold_high
-        and
-          slo_errors_per_request:ratio_rate5m{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"} > on (service) group_left slo_threshold_high
-        )
-        or
-        (
-          slo_errors_per_request:ratio_rate6h{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"} > on (service) group_left slo_threshold_low
-        and
-          slo_errors_per_request:ratio_rate30m{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"} > on (service) group_left slo_threshold_low
+        label_replace(
+            (
+                  slo_errors_per_request:ratio_rate1h{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"}
+                > on (service) group_left ()
+                  slo_threshold_high
+              and
+                  slo_errors_per_request:ratio_rate5m{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"}
+                > on (service) group_left ()
+                  slo_threshold_high
+            )
+          or
+            (
+                  slo_errors_per_request:ratio_rate6h{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"}
+                > on (service) group_left ()
+                  slo_threshold_low
+              and
+                  slo_errors_per_request:ratio_rate30m{service!~"efk-.*|.*external-dns.*|kong-.*|.*(ingress-nginx|nginx-ingress-controller).*"}
+                > on (service) group_left ()
+                  slo_threshold_low
+            ),
+          "team",
+          "$1",
+          "label_application_giantswarm_io_team",
+          "(.*)"
         )
       for: 5m
       labels:
@@ -35,4 +49,4 @@ spec:
         cancel_if_cluster_has_no_workers: "true"
         cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
-        team: '{{`{{ $labels.label_application_giantswarm_io_team }}`}}'
+        team: '{{`{{ $labels.team }}`}}'