Bump threshold for warning on operator memory usage from 60% to 70%. (#…

…193)
stackrox · Jan 24, 2024 · ba4c0d0 · ba4c0d0
1 parent 259c241
commit ba4c0d0
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/resources/prometheus/prometheus-rules.yaml b/resources/prometheus/prometheus-rules.yaml
@@ -214,7 +214,7 @@ spec:
           record: rhacs_operator:namespace:workload:container:max_memory_usage_ratio
         - alert: RHACSOperatorMemoryUtilizationHigh
           expr: |
-            rhacs_operator:namespace:workload:container:max_memory_usage_ratio > 0.6
+            rhacs_operator:namespace:workload:container:max_memory_usage_ratio > 0.7
           for: 5m
           labels:
             severity: warning

diff --git a/resources/prometheus/unit_tests/RHACSOperatorMemoryUtilizationHigh.yaml b/resources/prometheus/unit_tests/RHACSOperatorMemoryUtilizationHigh.yaml
@@ -11,7 +11,7 @@ tests:
       - series: kube_pod_labels{namespace="rhacs",label_app="rhacs-operator",pod="operator-pod"}
         values: "1+0x20"
       - series: container_memory_max_usage_bytes{namespace="rhacs", pod="operator-pod",container="manager"}
-        values: "50+0x10 70+0x10"
+        values: "50+0x10 80+0x10"
       - series: container_spec_memory_limit_bytes{namespace="rhacs",pod="operator-pod",container="manager"}
         values: "100+0x20"
     alert_rule_test:
@@ -28,7 +28,7 @@ tests:
               workload: operator-workload
               container: manager
             exp_annotations:
-              description: The container 'manager' in operator 'operator-workload' reached 70% of its memory limit and is at risk of being OOM killed.
+              description: The container 'manager' in operator 'operator-workload' reached 80% of its memory limit and is at risk of being OOM killed.
               summary: "The container 'manager' in operator 'operator-workload' is reaching its memory limit."
               sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-037-operator-memory-high.md"
   - interval: 1m