diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b357333..f8ad10cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - TBD ## New features -- TBD +- [Alertmanager] Add alertmanager resolve timeout configuration option - [#1187](https://github.com/jertel/elastalert2/pull/1187) - @eveningcafe ## Other changes - [Docs] Clarify docs to state that alert_time_limit should not be 0 - [#1208](https://github.com/jertel/elastalert2/pull/1208) - @jertel diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 9001aef6..abef7d24 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1828,7 +1828,9 @@ Optional: ``alertmanager_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``True`` if you want to ignore SSL errors. ``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles. - +`` +``alertmanager_resolve_time``: Optionally provide an automatic resolution timeframe. If no further alerts arrive within this time period alertmanager will automatically mark the alert as resolved. If not defined it will use Alertmanager's default behavior. +`` ``alertmanager_basic_auth_login``: Basic authentication username. ``alertmanager_basic_auth_password``: Basic authentication password. @@ -1842,6 +1844,8 @@ Example usage:: alertmanager_alertname: "Title" alertmanager_annotations: severity: "error" + alertmanager_resolve_time: + minutes: 10 alertmanager_labels: source: "elastalert" alertmanager_fields: diff --git a/elastalert/alerters/alertmanager.py b/elastalert/alerters/alertmanager.py index 70eb1598..05301a08 100644 --- a/elastalert/alerters/alertmanager.py +++ b/elastalert/alerters/alertmanager.py @@ -1,6 +1,7 @@ import json import warnings +from datetime import datetime, timedelta, timezone import requests from requests import RequestException from requests.auth import HTTPBasicAuth @@ -30,7 +31,10 @@ def __init__(self, rule): self.timeout = self.rule.get('alertmanager_timeout', 10) self.alertmanager_basic_auth_login = self.rule.get('alertmanager_basic_auth_login', None) self.alertmanager_basic_auth_password = self.rule.get('alertmanager_basic_auth_password', None) - + if 'alertmanager_resolve_time' in rule: + self.resolve_timeout = timedelta(**rule['alertmanager_resolve_time']) + else: + self.resolve_timeout = None @staticmethod def _json_or_string(obj): @@ -53,11 +57,16 @@ def alert(self, matches): self.annotations.update({ self.title_labelname: self.create_title(matches), self.body_labelname: self.create_alert_body(matches)}) + payload = { 'annotations': self.annotations, 'labels': self.labels } + if self.resolve_timeout is not None: + end_time = self.now() + self.resolve_timeout + payload['endsAt'] = end_time.isoformat() + for host in self.hosts: try: url = '{}/api/{}/alerts'.format(host, self.api_version) @@ -87,3 +96,6 @@ def alert(self, matches): def get_info(self): return {'type': 'alertmanager'} + + def now(self): + return datetime.nowdatetime.now(timezone.utc) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 186e8859..7590ada5 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -329,6 +329,7 @@ properties: alertmanager_ca_certs: {type: [boolean, string]} alertmanager_ignore_ssl_errors: {type: boolean} alertmanager_timeout: {type: integer} + alertmanager_resolve_time: *timedelta alertmanager_basic_auth_login: {type: string} alertmanager_basic_auth_password: {type: string} alertmanager_labels: diff --git a/tests/alerters/alertmanager_test.py b/tests/alerters/alertmanager_test.py index 5c831413..acb12c3b 100644 --- a/tests/alerters/alertmanager_test.py +++ b/tests/alerters/alertmanager_test.py @@ -1,3 +1,4 @@ +import datetime import json import logging import pytest @@ -386,3 +387,66 @@ def test_alertmanager_basic_auth(): auth=HTTPBasicAuth('user', 'password') ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_alertmanager_resolve_timeout(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_resolve_time': {'minutes': 10}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.now = mock.Mock(return_value=datetime.datetime(2023, 7, 14, tzinfo=datetime.timezone.utc)) + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'endsAt': '2023-07-14T00:10:00+00:00', + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10, + auth=None + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, "Alert sent to Alertmanager") == caplog.record_tuples[0]