sync: stage to production

.github/workflows/automerge.yaml

@@ -26,7 +26,7 @@ jobs:
     steps:
       - id: automerge
         name: automerge
-        uses: "pascalgn/[email protected].3"
+        uses: "pascalgn/[email protected].4"
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           MERGE_METHOD: merge

.github/workflows/main.yml

@@ -10,6 +10,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
       - uses: actions/setup-go@v5
         with:
           go-version: ">=1.18.0"

resources/prometheus/prometheus-rules.yaml

@@ -128,6 +128,22 @@ spec:
             summary: "Fleetshard synchronizer container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` restarted more than 3 times."
             description: "Fleetshard synchronizer container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` has restarted more than 3 times during the last 30 minutes."
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-005-fleetshard-sync-unavailable.md"
+        - alert: RHACSFleetshardCertificateExpiryCritical
+          expr: |
+            acs_fleetshard_certificate_expiration_timestamp <= 1 * 24 * 60 * 60 + time()
+          labels:
+            severity: critical
+          annotations:
+            summary: "Certificate expiring very soon: `{{ $labels.exported_namespace }}/{{ $labels.secret }}/{{ $labels.data_key }}`."
+            description: "Certificate `{{ $labels.exported_namespace }}/{{ $labels.secret }}/{{ $labels.data_key }}` expires on {{ humanizeTimestamp $value}}."
+        - alert: RHACSFleetshardCertificateExpiryWarning
+          expr: |
+            acs_fleetshard_certificate_expiration_timestamp <= 7* 24 * 60 * 60 + time()
+          labels:
+            severity: warning
+          annotations:
+            summary: "Certificate expiring soon: `{{ $labels.exported_namespace }}/{{ $labels.secret }}/{{ $labels.data_key }}`."
+            description: "Certificate `{{ $labels.exported_namespace }}/{{ $labels.secret }}/{{ $labels.data_key }}` expires on {{ humanizeTimestamp $value}}."
         - alert: RHACSFleetshardSyncReconciliationErrors
           expr: |
             acs_fleetshard_central_errors_per_reconciliations:ratio_rate10m > 0.10
@@ -225,8 +241,8 @@ spec:
           record: rhacs_tenants:namespace:pod:container:max_memory_usage_ratio
         - alert: RHACSTenantWorkloadMemoryUtilizationHigh
           expr: |
-            rhacs_tenants:namespace:pod:container:max_memory_usage_ratio{container="central"} >= 0.85
-          for: 10m
+            rhacs_tenants:namespace:pod:container:max_memory_usage_ratio{container="central"} >= 0.9
+          for: 30m
           labels:
             severity: warning
           annotations:

resources/prometheus/unit_tests/RHACSFleetschardCertificateExpiring.yaml

@@ -0,0 +1,51 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1d
+    input_series:
+      - series: acs_fleetshard_certificate_expiration_timestamp{exported_namespace="rhacs-00000000000000000000", secret="secret", data_key="key"}
+        values: "691200+0x15" # equals to 8 days
+
+    alert_rule_test:
+      - eval_time: 0
+        alertname: RHACSFleetshardCertificateExpiryWarning
+        exp_alerts: [ ]
+      - eval_time: 3d
+        alertname: RHACSFleetshardCertificateExpiryWarning
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSFleetshardCertificateExpiryWarning
+              exported_namespace: rhacs-00000000000000000000
+              secret: secret
+              data_key: key
+              severity: warning
+            exp_annotations:
+              summary: "Certificate expiring soon: `rhacs-00000000000000000000/secret/key`."
+              description: "Certificate `rhacs-00000000000000000000/secret/key` expires on 1970-01-09 00:00:00 +0000 UTC."
+      - eval_time: 7d
+        alertname: RHACSFleetshardCertificateExpiryCritical
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSFleetshardCertificateExpiryCritical
+              exported_namespace: rhacs-00000000000000000000
+              secret: secret
+              data_key: key
+              severity: critical
+            exp_annotations:
+              summary: "Certificate expiring very soon: `rhacs-00000000000000000000/secret/key`."
+              description: "Certificate `rhacs-00000000000000000000/secret/key` expires on 1970-01-09 00:00:00 +0000 UTC."
+      - eval_time: 10d
+        alertname: RHACSFleetshardCertificateExpiryCritical
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSFleetshardCertificateExpiryCritical
+              exported_namespace: rhacs-00000000000000000000
+              secret: secret
+              data_key: key
+              severity: critical
+            exp_annotations:
+              summary: "Certificate expiring very soon: `rhacs-00000000000000000000/secret/key`."
+              description: "Certificate `rhacs-00000000000000000000/secret/key` expires on 1970-01-09 00:00:00 +0000 UTC."

resources/prometheus/unit_tests/RHACSTenantWorkloadMemoryUtilizationHigh.yaml

@@ -7,14 +7,15 @@ tests:
   - interval: 1m
     input_series:
       - series: container_memory_working_set_bytes{namespace="rhacs-aaaaaaaaaaaaaaaaaaaa", pod="mypod", container="central"}
-        values: "50+0x10 85+0x10"
+        # first 10 minutes no alert and then 90% CPU usage for 40 minutes
+        values: "50+0x10 90+0x40"
       - series: container_spec_memory_limit_bytes{namespace="rhacs-aaaaaaaaaaaaaaaaaaaa",pod="mypod", container="central"}
-        values: "100+0x20"
+        values: "100+0x40"
     alert_rule_test:
       - eval_time: 1m
         alertname: RHACSTenantWorkloadMemoryUtilizationHigh
         exp_alerts: []
-      - eval_time: 21m
+      - eval_time: 41m
         alertname: RHACSTenantWorkloadMemoryUtilizationHigh
         exp_alerts:
           - exp_labels:
@@ -25,7 +26,7 @@ tests:
               container: central
             exp_annotations:
               summary: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' is reaching its memory limit.
-              description: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' reached 85% of its memory limit and is at risk of being OOM killed.
+              description: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' reached 90% of its memory limit and is at risk of being OOM killed.
               sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-039-tenant-workload-memory-utilization-high.md"
   - interval: 1m
     input_series: