package ramen alerts

VolumeSynchronizationDelay

Signed-off-by: rakeshgm <[email protected]>

config/prometheus/alerts.yaml

@@ -0,0 +1,32 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: k8s
+    role: alert-rules
+  name: alerting-rules
+  namespace: system
+spec: 
+  groups:
+    - name: alerts
+      rules:
+      - record: ramen_rpo_difference
+        expr: ramen_sync_duration_seconds{job="ramen-hub-operator-metrics-service"} / on(policyname, job) group_left() (ramen_policy_schedule_interval_seconds{job="ramen-hub-operator-metrics-service"})
+      - alert: VolumeSynchronizationDelay
+        expr: ramen_rpo_difference >= 3
+        for: 5s
+        labels:
+          cluster: "{{ $labels.cluster }}"
+          severity: critical
+        annotations:
+          description: "Syncing of volumes (DRPC: {{ $labels.obj_name }}, Namespace: {{ $labels.obj_namespace }}) is taking more than thrice the scheduled snapshot interval. This may cause data loss and a backlog of replication requests. To get around the delay, follow the instructions provided in the documentation."
+          alert_type: "DisasterRecovery"
+      - alert: VolumeSynchronizationDelay
+        expr: ramen_rpo_difference > 2 and ramen_rpo_difference < 3
+        for: 5s
+        labels:
+          cluster: "{{ $labels.cluster }}"
+          severity: warning
+        annotations:
+          description: "Syncing of volumes (DRPC: {{ $labels.obj_name }}, Namespace: {{ $labels.obj_namespace }}) is taking more than twice the scheduled snapshot interval. This may cause data loss and impact replication requests. Check the documentation for instructions on how to get around the delay."
+          alert_type: "DisasterRecovery"

config/prometheus/kustomization.yaml

@@ -1,2 +1,3 @@
 resources:
 - monitor.yaml
+- alerts.yaml