Merge pull request #265 from stackrox/mc/increase-np-alerting-threshold

Increase alerting threshold for networt policy violations
stackrox · Jul 4, 2024 · 164fa71 · 164fa71
2 parents 5fa01d6 + 45f3e45
commit 164fa71
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 33 deletions.
diff --git a/resources/prometheus/prometheus-rules.yaml b/resources/prometheus/prometheus-rules.yaml
@@ -780,18 +780,7 @@ for the cluster autoscaler. Limits can be adjusted by modifying the cluster auto
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-043-selinux-violation.md"
         - alert: ClusterAuditNetworkPolicyViolations
           expr: |
-            network_policy_denials_sample_count > 0
-          for: 10m
-          labels:
-            severity: info
-          annotations:
-            summary: "Network Policy Violations occuring on cluster."
-            description: |
-              A cluster node logged Network Policy ACL denial(s) for 10 minutes.
-            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
-        - alert: ClusterAuditNetworkPolicyViolations
-          expr: |
-            network_policy_denials_sample_count >= 15
+            network_policy_denials_sample_count >= 25
           for: 1m
           labels:
             severity: info

diff --git a/resources/prometheus/unit_tests/ClusterAuditNetworkPolicyViolations.yaml b/resources/prometheus/unit_tests/ClusterAuditNetworkPolicyViolations.yaml
@@ -7,7 +7,7 @@ tests:
   - interval: 1m
     input_series:
       - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
-        values: "15x1"
+        values: "25x1"
     alert_rule_test:
       - eval_time: 70s
         alertname: ClusterAuditNetworkPolicyViolations
@@ -19,25 +19,7 @@ tests:
             exp_annotations:
               summary: "Network Policy Violations occuring on cluster."
               description: |
-                A cluster node logged at least 15 Network Policy ACL denial(s) per minute.
-              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
-
-  - interval: 1m
-    input_series:
-      - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
-        values: "1x10"
-    alert_rule_test:
-      - eval_time: 610s
-        alertname: ClusterAuditNetworkPolicyViolations
-        exp_alerts:
-          - exp_labels:
-              alertname: ClusterAuditNetworkPolicyViolations
-              namespace: rhacs-cloudwatch
-              severity: info
-            exp_annotations:
-              summary: "Network Policy Violations occuring on cluster."
-              description: |
-                A cluster node logged Network Policy ACL denial(s) for 10 minutes.
+                A cluster node logged at least 25 Network Policy ACL denial(s) per minute.
               sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
 
   - interval: 1m
@@ -52,7 +34,7 @@ tests:
   - interval: 1m
     input_series:
       - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
-        values: "14x1"
+        values: "24x1"
     alert_rule_test:
       - eval_time: 70s
         alertname: ClusterAuditNetworkPolicyViolations