sync: stage to production

resources/grafana/sources/rhacs-central-slo.json

@@ -26,7 +26,7 @@
         },
         "enable": true,
         "iconColor": "purple",
-        "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1",
+        "expr": "count (count by (git_version) (label_replace(count_over_time(kubernetes_build_info{job!~\"kube-dns|coredns\"}[${__interval}]), \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1",
         "name": "Kubernetes Upgrade",
         "textFormat": "Kubernetes Upgrade"
       },
@@ -37,7 +37,7 @@
         },
         "enable": true,
         "iconColor": "red",
-        "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1",
+        "expr": "count (count by (gitVersion) (count_over_time (openshift_apiserver_build_info[${__interval}]))) > 1",
         "name": "OpenShift Upgrade",
         "textFormat": "OpenShift Upgrade"
       }

resources/grafana/sources/rhacs-cluster-overview.json

@@ -26,7 +26,7 @@
         },
         "enable": true,
         "iconColor": "purple",
-        "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1",
+        "expr": "count (count by (git_version) (label_replace(count_over_time(kubernetes_build_info{job!~\"kube-dns|coredns\"}[${__interval}]), \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1",
         "name": "Kubernetes Upgrade",
         "textFormat": "Kubernetes Upgrade"
       },
@@ -37,7 +37,7 @@
         },
         "enable": true,
         "iconColor": "red",
-        "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1",
+        "expr": "count (count by (gitVersion) (count_over_time (openshift_apiserver_build_info[${__interval}]))) > 1",
         "name": "OpenShift Upgrade",
         "textFormat": "OpenShift Upgrade"
       }

resources/prometheus/prometheus-rules.yaml

@@ -756,3 +756,35 @@ cluster autoscaler. Limits can be adjusted by modifying the cluster autoscaler c
 the cluster autoscaler. This is calculated by summing the memory capacity for all nodes in the cluster and comparing that number against the maximum memory bytes value set
 for the cluster autoscaler. Limits can be adjusted by modifying the cluster autoscaler configuration."
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-042-modify-cluster-autoscaler.md"
+        - alert: ClusterAuditSELinuxViolations
+          expr: |
+            selinux_denials_sample_count > 0
+          labels:
+            severity: info
+          annotations:
+            summary: "SELinux Violations occuring on cluster."
+            description: |
+              A cluster node logged {{ $value }} SELinux AVC denial(s) per minute to the audit log.
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-043-selinux-violation.md"
+        - alert: ClusterAuditNetworkPolicyViolations
+          expr: |
+            network_policy_denials_sample_count > 0
+          for: 10m
+          labels:
+            severity: info
+          annotations:
+            summary: "Network Policy Violations occuring on cluster."
+            description: |
+              A cluster node logged Network Policy ACL denial(s) for 10 minutes.
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
+        - alert: ClusterAuditNetworkPolicyViolations
+          expr: |
+            network_policy_denials_sample_count >= 15
+          for: 1m
+          labels:
+            severity: info
+          annotations:
+            summary: "Network Policy Violations occuring on cluster."
+            description: |
+              A cluster node logged at least {{ $value }} Network Policy ACL denial(s) per minute.
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"

resources/prometheus/unit_tests/ClusterAuditNetworkPolicyViolations.yaml

@@ -0,0 +1,59 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
+        values: "15x1"
+    alert_rule_test:
+      - eval_time: 70s
+        alertname: ClusterAuditNetworkPolicyViolations
+        exp_alerts:
+          - exp_labels:
+              alertname: ClusterAuditNetworkPolicyViolations
+              namespace: rhacs-cloudwatch
+              severity: info
+            exp_annotations:
+              summary: "Network Policy Violations occuring on cluster."
+              description: |
+                A cluster node logged at least 15 Network Policy ACL denial(s) per minute.
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
+
+  - interval: 1m
+    input_series:
+      - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
+        values: "1x10"
+    alert_rule_test:
+      - eval_time: 610s
+        alertname: ClusterAuditNetworkPolicyViolations
+        exp_alerts:
+          - exp_labels:
+              alertname: ClusterAuditNetworkPolicyViolations
+              namespace: rhacs-cloudwatch
+              severity: info
+            exp_annotations:
+              summary: "Network Policy Violations occuring on cluster."
+              description: |
+                A cluster node logged Network Policy ACL denial(s) for 10 minutes.
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-044-network-policy-violation.md"
+
+  - interval: 1m
+    input_series:
+      - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
+        values: "1x9 0"
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: ClusterAuditNetworkPolicyViolations
+        exp_alerts: []
+
+  - interval: 1m
+    input_series:
+      - series: network_policy_denials_sample_count{namespace="rhacs-cloudwatch"}
+        values: "14x1"
+    alert_rule_test:
+      - eval_time: 70s
+        alertname: ClusterAuditNetworkPolicyViolations
+        exp_alerts: []

resources/prometheus/unit_tests/ClusterAuditSELinuxViolations.yaml

@@ -0,0 +1,23 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: selinux_denials_sample_count{namespace="rhacs-cloudwatch"}
+        values: "1x5"
+    alert_rule_test:
+      - eval_time: 70s
+        alertname: ClusterAuditSELinuxViolations
+        exp_alerts:
+          - exp_labels:
+              alertname: ClusterAuditSELinuxViolations
+              namespace: rhacs-cloudwatch
+              severity: info
+            exp_annotations:
+              summary: "SELinux Violations occuring on cluster."
+              description: |
+                A cluster node logged 1 SELinux AVC denial(s) per minute to the audit log.
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-043-selinux-violation.md"