Add alert rules to kfp-api based on the KF093 spec (#560) (#575)

* Add alert rules to kfp-api based on the KF093 spec * Delete charms/kfp-api/src/prometheus_alert_rules/unit_unavailable.rule Co-authored-by: Robert Gildein <[email protected]>
canonical · Oct 11, 2024 · 1b8e9e3 · 1b8e9e3
1 parent e9f1ad5
commit 1b8e9e3
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 10 deletions.
diff --git a/charms/kfp-api/src/prometheus_alert_rules/KubeflowKfpApiServices.rules b/charms/kfp-api/src/prometheus_alert_rules/KubeflowKfpApiServices.rules
@@ -0,0 +1,24 @@
+groups:
+- name: KubeflowKfpApiServices
+  rules:
+  - alert: KubeflowServiceDown
+    expr: up{} < 1
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is Down ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+       One or more targets of {{ $labels.juju_charm }} charm are down on unit {{ $labels.juju_model }}/{{ $labels.juju_unit }}.
+       LABELS = {{ $labels }}
+
+  - alert: KubeflowServiceIsNotStable
+    expr: avg_over_time(up{}[10m]) < 0.5
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is not stable ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+        {{ $labels.juju_charm }} unit {{ $labels.juju_model }}/{{ $labels.juju_unit }} has been unreachable at least 50% of the time over the last 10 minutes.
+        LABELS = {{ $labels }}
diff --git a/charms/kfp-api/src/prometheus_alert_rules/unit_unavailable.rule b/charms/kfp-api/src/prometheus_alert_rules/unit_unavailable.rule