From fe11d851fe0131c0cfb0bbbe481ab3134d021e74 Mon Sep 17 00:00:00 2001
From: Herve Nicol <12008875+hervenicol@users.noreply.github.com>
Date: Tue, 12 Mar 2024 11:04:51 +0100
Subject: [PATCH 1/2] Set PromtailDown alert to not page out of business hours

---
 CHANGELOG.md                                                  | 4 ++++
 .../templates/alerting-rules/promtail.rules.yml               | 1 +
 2 files changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 402e6106b..d142eba0e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Set `PromtailDown` alert to not page out of business hours
+
 ### Fixed
 
 - Fix `AWSLoadBalancerControllerReconcileErrors` alert query.
diff --git a/helm/prometheus-rules/templates/alerting-rules/promtail.rules.yml b/helm/prometheus-rules/templates/alerting-rules/promtail.rules.yml
index f11abe93b..e29bab951 100644
--- a/helm/prometheus-rules/templates/alerting-rules/promtail.rules.yml
+++ b/helm/prometheus-rules/templates/alerting-rules/promtail.rules.yml
@@ -20,6 +20,7 @@ spec:
             severity: page
             team: "atlas"
             topic: "observability"
+            cancel_if_outside_working_hours: "true"
             cancel_if_cluster_status_creating: "true"
             cancel_if_cluster_status_deleting: "true"
             cancel_if_cluster_status_updating: "true"

From 36f45b222cfe559d25259febca8edca7c17696eb Mon Sep 17 00:00:00 2001
From: Herve Nicol <12008875+hervenicol@users.noreply.github.com>
Date: Tue, 12 Mar 2024 11:13:06 +0100
Subject: [PATCH 2/2] fix tests

---
 test/tests/providers/global/promtail.rules.test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/tests/providers/global/promtail.rules.test.yml b/test/tests/providers/global/promtail.rules.test.yml
index 724a4b6a7..cc6a2efcc 100644
--- a/test/tests/providers/global/promtail.rules.test.yml
+++ b/test/tests/providers/global/promtail.rules.test.yml
@@ -23,6 +23,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: empowerment
+              cancel_if_outside_working_hours: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
               cancel_if_cluster_status_updating: "true"
@@ -39,6 +40,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: empowerment
+              cancel_if_outside_working_hours: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
               cancel_if_cluster_status_updating: "true"
@@ -56,6 +58,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: empowerment
+              cancel_if_outside_working_hours: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
               cancel_if_cluster_status_updating: "true"