diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4ef31084..651e607e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1810,7 +1810,7 @@ Optional: ``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles. `` -``alertmanager_resolve_timeout``: You can specify a timeout value, in minutes.The default is 5, and we recommend leaving this configuration greater than the ``buffer_time`` or the ``realert`` value because Alertmanager has a default ``resolve_timeout`` of 5 minutes (this means that after 5 minutes, if it doesn't receive any repeated alerts, it will consider the alert to have stopped firing). +``alertmanager_resolve_time``: Optionally provide an automatic resolution timeframe. If no further alerts arrive within this time period alertmanager will automatically mark the alert as resolved. If not defined it will use Alertmanager's default behavior. `` ``alertmanager_basic_auth_login``: Basic authentication username. @@ -1825,6 +1825,8 @@ Example usage:: alertmanager_alertname: "Title" alertmanager_annotations: severity: "error" + alertmanager_resolve_time: + minutes: 10 alertmanager_labels: source: "elastalert" alertmanager_fields: diff --git a/elastalert/alerters/alertmanager.py b/elastalert/alerters/alertmanager.py index 7791b3c4..05301a08 100644 --- a/elastalert/alerters/alertmanager.py +++ b/elastalert/alerters/alertmanager.py @@ -22,7 +22,6 @@ def __init__(self, rule): self.alertname = self.rule.get('alertmanager_alertname', self.rule.get('name')) self.labels = self.rule.get('alertmanager_labels', dict()) self.annotations = self.rule.get('alertmanager_annotations', dict()) - self.resolve_timeout = self.rule.get('alertmanager_resolve_timeout', 5) self.fields = self.rule.get('alertmanager_fields', dict()) self.title_labelname = self.rule.get('alertmanager_alert_subject_labelname', 'summary') self.body_labelname = self.rule.get('alertmanager_alert_text_labelname', 'description') @@ -32,7 +31,10 @@ def __init__(self, rule): self.timeout = self.rule.get('alertmanager_timeout', 10) self.alertmanager_basic_auth_login = self.rule.get('alertmanager_basic_auth_login', None) self.alertmanager_basic_auth_password = self.rule.get('alertmanager_basic_auth_password', None) - + if 'alertmanager_resolve_time' in rule: + self.resolve_timeout = timedelta(**rule['alertmanager_resolve_time']) + else: + self.resolve_timeout = None @staticmethod def _json_or_string(obj): @@ -55,14 +57,16 @@ def alert(self, matches): self.annotations.update({ self.title_labelname: self.create_title(matches), self.body_labelname: self.create_alert_body(matches)}) - end_time = datetime.now(timezone.utc) + timedelta(minutes=self.resolve_timeout) payload = { 'annotations': self.annotations, - 'labels': self.labels, - 'endsAt': end_time.isoformat() + 'labels': self.labels } + if self.resolve_timeout is not None: + end_time = self.now() + self.resolve_timeout + payload['endsAt'] = end_time.isoformat() + for host in self.hosts: try: url = '{}/api/{}/alerts'.format(host, self.api_version) @@ -92,3 +96,6 @@ def alert(self, matches): def get_info(self): return {'type': 'alertmanager'} + + def now(self): + return datetime.nowdatetime.now(timezone.utc) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4b08fc3c..8e2705f1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -328,9 +328,7 @@ properties: alertmanager_ca_certs: {type: [boolean, string]} alertmanager_ignore_ssl_errors: {type: boolean} alertmanager_timeout: {type: integer} - alertmanager_resolve_timeout: - type: integer - minProperties: 1 + alertmanager_resolve_time: *timedelta alertmanager_basic_auth_login: {type: string} alertmanager_basic_auth_password: {type: string} alertmanager_labels: diff --git a/tests/alerters/alertmanager_test.py b/tests/alerters/alertmanager_test.py index 5c831413..acb12c3b 100644 --- a/tests/alerters/alertmanager_test.py +++ b/tests/alerters/alertmanager_test.py @@ -1,3 +1,4 @@ +import datetime import json import logging import pytest @@ -386,3 +387,66 @@ def test_alertmanager_basic_auth(): auth=HTTPBasicAuth('user', 'password') ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_alertmanager_resolve_timeout(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_resolve_time': {'minutes': 10}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.now = mock.Mock(return_value=datetime.datetime(2023, 7, 14, tzinfo=datetime.timezone.utc)) + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'endsAt': '2023-07-14T00:10:00+00:00', + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10, + auth=None + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, "Alert sent to Alertmanager") == caplog.record_tuples[0]