Add alerts to monitor the HelmReleases for cilium and coredns

giantswarm · Nov 19, 2024 · b54b0d7 · b54b0d7
1 parent 4ac59b0
commit b54b0d7
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add alerts to monitor the `HelmReleases` for `cilium` and `coredns`.
+
 ## [4.26.1] - 2024-11-19
 
 ### Changed

diff --git a/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/cilium.rules.yml b/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/cilium.rules.yml
@@ -59,4 +59,20 @@ spec:
         severity: page
         team: cabbage
         topic: cilium
-
+    - alert: FluxHelmReleaseFailed
+      annotations:
+        description: |-
+          {{`Flux HelmRelease {{ $labels.name }} in ns {{ $labels.exported_namespace }} on {{ $labels.installation }}/{{ $labels.cluster_id }} is stuck in Failed state.`}}
+        opsrecipe: fluxcd-failing-helmrelease/
+      expr: gotk_reconcile_condition{type="Ready", status="False", kind="HelmRelease", cluster_type="management_cluster", exported_namespace!="flux-giantswarm", name=~"cilium"} > 0
+      for: 20m
+      labels:
+        area: platform
+        cancel_if_outside_working_hours: "true"
+        cancel_if_kube_state_metrics_down: "true"
+        cancel_if_monitoring_agent_down: "true"
+        severity: page
+        team: cabbage
+        topic: cilium
+        namespace: |-
+          {{`{{ $labels.exported_namespace }}`}}
diff --git a/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/coredns.rules.yml b/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/coredns.rules.yml
@@ -25,6 +25,23 @@ spec:
         severity: page
         team: cabbage
         topic: dns
+    - alert: FluxHelmReleaseFailed
+      annotations:
+        description: |-
+          {{`Flux HelmRelease {{ $labels.name }} in ns {{ $labels.exported_namespace }} on {{ $labels.installation }}/{{ $labels.cluster_id }} is stuck in Failed state.`}}
+        opsrecipe: fluxcd-failing-helmrelease/
+      expr: gotk_reconcile_condition{type="Ready", status="False", kind="HelmRelease", cluster_type="management_cluster", exported_namespace!="flux-giantswarm", name=~"coredns"} > 0
+      for: 20m
+      labels:
+        area: platform
+        cancel_if_outside_working_hours: "true"
+        cancel_if_kube_state_metrics_down: "true"
+        cancel_if_monitoring_agent_down: "true"
+        severity: page
+        team: cabbage
+        topic: dns
+        namespace: |-
+          {{`{{ $labels.exported_namespace }}`}}
     - alert: CoreDNSMaxHPAReplicasReached
       expr: |
         (