ROX-20792: Add critical alert for operator OOM

resources/prometheus/prometheus-rules.yaml

@@ -162,33 +162,46 @@ spec:
     - name: rhacs-operator
       rules:
         - expr: |
-            sum (kube_pod_info{namespace="rhacs"}
-            * on (pod, namespace) group_left() kube_pod_labels{namespace="rhacs", label_app="rhacs-operator"}
-            * on (pod, namespace) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel) by (namespace, workload, pod)
-          record: rhacs_operator:namespace:workload:pod
+            sum (namespace_workload_pod:kube_pod_owner:relabel{namespace="rhacs"}
+            * on (pod, namespace) group_left() kube_pod_labels{namespace="rhacs", label_app="rhacs-operator"})
+            by (pod, namespace, workload)
+          record: rhacs_operator:namespace:workload:pod:container
         - expr: |
-            rhacs_operator:namespace:workload:pod
-            * on (pod, namespace) group_left() sum(container_memory_max_usage_bytes{container!=""}) by (pod, namespace)
-          record: rhacs_operator:namespace:workload:pod:max_memory_usage_bytes
+            sum(container_memory_max_usage_bytes{namespace="rhacs",container!~"POD|"}) by (container, pod, namespace)
+            * on (namespace, pod) group_left(workload) rhacs_operator:namespace:workload:pod:container
+          record: rhacs_operator:namespace:workload:pod:container:max_memory_usage_bytes
         - expr: |
-            rhacs_operator:namespace:workload:pod
-            * on (pod, namespace) group_left() sum(container_spec_memory_limit_bytes{container!=""}) by (pod, namespace)
-          record: rhacs_operator:namespace:workload:pod:memory_limit_bytes
+            sum(container_spec_memory_limit_bytes{namespace="rhacs",container!~"POD|"}) by (container, pod, namespace)
+            * on (namespace, pod) group_left(workload) rhacs_operator:namespace:workload:pod:container
+          record: rhacs_operator:namespace:workload:pod:container:memory_limit_bytes
         - expr: |
-            sum(rhacs_operator:namespace:workload:pod:max_memory_usage_bytes / rhacs_operator:namespace:workload:pod:memory_limit_bytes)
-            by (namespace, workload)
-          record: rhacs_operator:namespace:workload:max_memory_usage_ratio
+            sum(
+              rhacs_operator:namespace:workload:pod:container:max_memory_usage_bytes
+            / rhacs_operator:namespace:workload:pod:container:memory_limit_bytes)
+            by (namespace, workload, container)
+          record: rhacs_operator:namespace:workload:container:max_memory_usage_ratio
         - alert: RHACSOperatorMemoryUtilizationHigh
           expr: |
-            rhacs_operator:namespace:workload:max_memory_usage_ratio > 0.6
+            rhacs_operator:namespace:workload:container:max_memory_usage_ratio > 0.6
           for: 5m
           labels:
             severity: warning
           annotations:
-            summary: RHACS Operator '{{ $labels.workload }}' is reaching its memory limit.
-            description: The RHACS operator '{{ $labels.workload }}' reached {{ $value | humanizePercentage }} of its memory limit and is at risk of being OOM killed.
+            summary: The container '{{ $labels.container }}' in operator '{{ $labels.workload }}' is reaching its memory limit.
+            description: The container '{{ $labels.container }}' in operator '{{ $labels.workload }}' reached {{ $value | humanizePercentage }} of its memory limit and is at risk of being OOM killed.
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-037-operator-memory-high.md"
+        - alert: RHACSOperatorMemoryUtilizationCritical
+          expr: |
+            rhacs_operator:namespace:workload:container:max_memory_usage_ratio > 0.9
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: The container '{{ $labels.container }}' in operator '{{ $labels.workload }}' is critically reaching its memory limit.
+            description: The container '{{ $labels.container }}' in operator '{{ $labels.workload }}' reached {{ $value | humanizePercentage }} of its memory limit and is at high risk of being OOM killed.
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-037-operator-memory-high.md"
 
+
     - name: rhacs-aws-quota
       rules:
         - alert: RHACSCentralDBClustersUtilizationHigh

resources/prometheus/unit_tests/RHACSOperatorMemoryUtilizationHigh.yaml

@@ -6,15 +6,13 @@ evaluation_interval: 1m
 tests:
   - interval: 1m
     input_series:
-      - series: kube_pod_info{namespace="rhacs", pod="operator-pod"}
+      - series: namespace_workload_pod:kube_pod_owner:relabel{namespace="rhacs",workload="operator-workload",pod="operator-pod"}
         values: "1+0x20"
-      - series: kube_pod_labels{namespace="rhacs", pod="operator-pod", label_app="rhacs-operator"}
+      - series: kube_pod_labels{namespace="rhacs",label_app="rhacs-operator",pod="operator-pod"}
         values: "1+0x20"
-      - series: namespace_workload_pod:kube_pod_owner:relabel{namespace="rhacs", pod="operator-pod", workload="operator-workload"}
-        values: "1+0x20"
-      - series: container_memory_max_usage_bytes{namespace="rhacs", pod="operator-pod", container="operator-container"}
+      - series: container_memory_max_usage_bytes{namespace="rhacs", pod="operator-pod",container="manager"}
         values: "50+0x10 70+0x10"
-      - series: container_spec_memory_limit_bytes{namespace="rhacs", pod="operator-pod", container="operator-container"}
+      - series: container_spec_memory_limit_bytes{namespace="rhacs",pod="operator-pod",container="manager"}
         values: "100+0x20"
     alert_rule_test:
       - eval_time: 1m
@@ -28,7 +26,35 @@ tests:
               severity: warning
               namespace: rhacs
               workload: operator-workload
+              container: manager
+            exp_annotations:
+              description: The container 'manager' in operator 'operator-workload' reached 70% of its memory limit and is at risk of being OOM killed.
+              summary: "The container 'manager' in operator 'operator-workload' is reaching its memory limit."
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-037-operator-memory-high.md"
+  - interval: 1m
+    input_series:
+      - series: namespace_workload_pod:kube_pod_owner:relabel{namespace="rhacs",workload="operator-workload",pod="operator-pod"}
+        values: "1+0x20"
+      - series: kube_pod_labels{namespace="rhacs",label_app="rhacs-operator",pod="operator-pod"}
+        values: "1+0x20"
+      - series: container_memory_max_usage_bytes{namespace="rhacs", pod="operator-pod",container="manager"}
+        values: "50+0x10 91+0x10"
+      - series: container_spec_memory_limit_bytes{namespace="rhacs",pod="operator-pod",container="manager"}
+        values: "100+0x20"
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: RHACSOperatorMemoryUtilizationCritical
+        exp_alerts: []
+      - eval_time: 17m
+        alertname: RHACSOperatorMemoryUtilizationCritical
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSOperatorMemoryUtilizationCritical
+              severity: critical
+              namespace: rhacs
+              workload: operator-workload
+              container: manager
             exp_annotations:
-              description: "The RHACS operator 'operator-workload' reached 70% of its memory limit and is at risk of being OOM killed."
-              summary: "RHACS Operator 'operator-workload' is reaching its memory limit."
+              description: The container 'manager' in operator 'operator-workload' reached 91% of its memory limit and is at high risk of being OOM killed.
+              summary: The container 'manager' in operator 'operator-workload' is critically reaching its memory limit.
               sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-037-operator-memory-high.md"