From cb86ef6bcc8fb8cd7bd0e0c6690c9b018d056f7d Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 May 2023 17:59:15 +0700 Subject: [PATCH 1/4] add_alertmanager_resolve_timeout --- elastalert/alerters/alertmanager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elastalert/alerters/alertmanager.py b/elastalert/alerters/alertmanager.py index 70eb1598..7791b3c4 100644 --- a/elastalert/alerters/alertmanager.py +++ b/elastalert/alerters/alertmanager.py @@ -1,6 +1,7 @@ import json import warnings +from datetime import datetime, timedelta, timezone import requests from requests import RequestException from requests.auth import HTTPBasicAuth @@ -21,6 +22,7 @@ def __init__(self, rule): self.alertname = self.rule.get('alertmanager_alertname', self.rule.get('name')) self.labels = self.rule.get('alertmanager_labels', dict()) self.annotations = self.rule.get('alertmanager_annotations', dict()) + self.resolve_timeout = self.rule.get('alertmanager_resolve_timeout', 5) self.fields = self.rule.get('alertmanager_fields', dict()) self.title_labelname = self.rule.get('alertmanager_alert_subject_labelname', 'summary') self.body_labelname = self.rule.get('alertmanager_alert_text_labelname', 'description') @@ -53,9 +55,12 @@ def alert(self, matches): self.annotations.update({ self.title_labelname: self.create_title(matches), self.body_labelname: self.create_alert_body(matches)}) + end_time = datetime.now(timezone.utc) + timedelta(minutes=self.resolve_timeout) + payload = { 'annotations': self.annotations, - 'labels': self.labels + 'labels': self.labels, + 'endsAt': end_time.isoformat() } for host in self.hosts: From f8a4aea41699451e595b9ed5efcaf7739bc59ef4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 13 Jul 2023 17:26:51 +0700 Subject: [PATCH 2/4] add changelog --- CHANGELOG.md | 2 +- docs/source/ruletypes.rst | 4 +++- elastalert/schema.yaml | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27024b13..69267fb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ - [Kubernetes] Fix mistake introduced in #1141 related to initContainers - [#1145](https://github.com/jertel/elastalert2/pull/1145) - @aturpin1789 - Add support for Kibana 8.7 for Kibana Discover - [#1153](https://github.com/jertel/elastalert2/pull/1153) - @nsano-rururu - [Docs] Add documentation for accessing subfields of array items - [#1166](https://github.com/jertel/elastalert2/pull/1166) - @jertel - +- Add alertmanager resolve timeout configuration option. # 2.10.1 ## Breaking changes diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 80cc3486..4ef31084 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1809,7 +1809,9 @@ Optional: ``alertmanager_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``True`` if you want to ignore SSL errors. ``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles. - +`` +``alertmanager_resolve_timeout``: You can specify a timeout value, in minutes.The default is 5, and we recommend leaving this configuration greater than the ``buffer_time`` or the ``realert`` value because Alertmanager has a default ``resolve_timeout`` of 5 minutes (this means that after 5 minutes, if it doesn't receive any repeated alerts, it will consider the alert to have stopped firing). +`` ``alertmanager_basic_auth_login``: Basic authentication username. ``alertmanager_basic_auth_password``: Basic authentication password. diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index f3f46051..4b08fc3c 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -328,6 +328,9 @@ properties: alertmanager_ca_certs: {type: [boolean, string]} alertmanager_ignore_ssl_errors: {type: boolean} alertmanager_timeout: {type: integer} + alertmanager_resolve_timeout: + type: integer + minProperties: 1 alertmanager_basic_auth_login: {type: string} alertmanager_basic_auth_password: {type: string} alertmanager_labels: From 0bb8f44639a16d5c1244c0a55a62049ad8f83fa8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 13 Jul 2023 07:27:00 -0400 Subject: [PATCH 3/4] correct changelog entry --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69267fb7..dbdf9338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - TBD ## New features -- TBD +- [Alertmanager] Add alertmanager resolve timeout configuration option - [#1187](https://github.com/jertel/elastalert2/pull/1187) - @eveningcafe ## Other changes - TBD @@ -21,7 +21,7 @@ - [Kubernetes] Fix mistake introduced in #1141 related to initContainers - [#1145](https://github.com/jertel/elastalert2/pull/1145) - @aturpin1789 - Add support for Kibana 8.7 for Kibana Discover - [#1153](https://github.com/jertel/elastalert2/pull/1153) - @nsano-rururu - [Docs] Add documentation for accessing subfields of array items - [#1166](https://github.com/jertel/elastalert2/pull/1166) - @jertel -- Add alertmanager resolve timeout configuration option. + # 2.10.1 ## Breaking changes From c5f30128aece7f0d8ad05df692d1dc793e849ca5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 14 Jul 2023 08:10:22 -0400 Subject: [PATCH 4/4] refactor implementation and add unit test --- docs/source/ruletypes.rst | 4 +- elastalert/alerters/alertmanager.py | 17 +++++--- elastalert/schema.yaml | 4 +- tests/alerters/alertmanager_test.py | 64 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4ef31084..651e607e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1810,7 +1810,7 @@ Optional: ``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles. `` -``alertmanager_resolve_timeout``: You can specify a timeout value, in minutes.The default is 5, and we recommend leaving this configuration greater than the ``buffer_time`` or the ``realert`` value because Alertmanager has a default ``resolve_timeout`` of 5 minutes (this means that after 5 minutes, if it doesn't receive any repeated alerts, it will consider the alert to have stopped firing). +``alertmanager_resolve_time``: Optionally provide an automatic resolution timeframe. If no further alerts arrive within this time period alertmanager will automatically mark the alert as resolved. If not defined it will use Alertmanager's default behavior. `` ``alertmanager_basic_auth_login``: Basic authentication username. @@ -1825,6 +1825,8 @@ Example usage:: alertmanager_alertname: "Title" alertmanager_annotations: severity: "error" + alertmanager_resolve_time: + minutes: 10 alertmanager_labels: source: "elastalert" alertmanager_fields: diff --git a/elastalert/alerters/alertmanager.py b/elastalert/alerters/alertmanager.py index 7791b3c4..05301a08 100644 --- a/elastalert/alerters/alertmanager.py +++ b/elastalert/alerters/alertmanager.py @@ -22,7 +22,6 @@ def __init__(self, rule): self.alertname = self.rule.get('alertmanager_alertname', self.rule.get('name')) self.labels = self.rule.get('alertmanager_labels', dict()) self.annotations = self.rule.get('alertmanager_annotations', dict()) - self.resolve_timeout = self.rule.get('alertmanager_resolve_timeout', 5) self.fields = self.rule.get('alertmanager_fields', dict()) self.title_labelname = self.rule.get('alertmanager_alert_subject_labelname', 'summary') self.body_labelname = self.rule.get('alertmanager_alert_text_labelname', 'description') @@ -32,7 +31,10 @@ def __init__(self, rule): self.timeout = self.rule.get('alertmanager_timeout', 10) self.alertmanager_basic_auth_login = self.rule.get('alertmanager_basic_auth_login', None) self.alertmanager_basic_auth_password = self.rule.get('alertmanager_basic_auth_password', None) - + if 'alertmanager_resolve_time' in rule: + self.resolve_timeout = timedelta(**rule['alertmanager_resolve_time']) + else: + self.resolve_timeout = None @staticmethod def _json_or_string(obj): @@ -55,14 +57,16 @@ def alert(self, matches): self.annotations.update({ self.title_labelname: self.create_title(matches), self.body_labelname: self.create_alert_body(matches)}) - end_time = datetime.now(timezone.utc) + timedelta(minutes=self.resolve_timeout) payload = { 'annotations': self.annotations, - 'labels': self.labels, - 'endsAt': end_time.isoformat() + 'labels': self.labels } + if self.resolve_timeout is not None: + end_time = self.now() + self.resolve_timeout + payload['endsAt'] = end_time.isoformat() + for host in self.hosts: try: url = '{}/api/{}/alerts'.format(host, self.api_version) @@ -92,3 +96,6 @@ def alert(self, matches): def get_info(self): return {'type': 'alertmanager'} + + def now(self): + return datetime.nowdatetime.now(timezone.utc) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4b08fc3c..8e2705f1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -328,9 +328,7 @@ properties: alertmanager_ca_certs: {type: [boolean, string]} alertmanager_ignore_ssl_errors: {type: boolean} alertmanager_timeout: {type: integer} - alertmanager_resolve_timeout: - type: integer - minProperties: 1 + alertmanager_resolve_time: *timedelta alertmanager_basic_auth_login: {type: string} alertmanager_basic_auth_password: {type: string} alertmanager_labels: diff --git a/tests/alerters/alertmanager_test.py b/tests/alerters/alertmanager_test.py index 5c831413..acb12c3b 100644 --- a/tests/alerters/alertmanager_test.py +++ b/tests/alerters/alertmanager_test.py @@ -1,3 +1,4 @@ +import datetime import json import logging import pytest @@ -386,3 +387,66 @@ def test_alertmanager_basic_auth(): auth=HTTPBasicAuth('user', 'password') ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_alertmanager_resolve_timeout(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_resolve_time': {'minutes': 10}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.now = mock.Mock(return_value=datetime.datetime(2023, 7, 14, tzinfo=datetime.timezone.utc)) + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'endsAt': '2023-07-14T00:10:00+00:00', + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10, + auth=None + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, "Alert sent to Alertmanager") == caplog.record_tuples[0]