From 54de9666d28dff36ee2e0f7af979202e7ef44313 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 9 Oct 2021 00:16:03 +0900 Subject: [PATCH 1/2] Add Alertmanager alerter --- docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 49 +++++ elastalert/alerters/alertmanager.py | 83 +++++++ elastalert/loaders.py | 2 + elastalert/schema.yaml | 46 ++++ tests/alerters/alertmanager_test.py | 324 ++++++++++++++++++++++++++++ 6 files changed, 505 insertions(+) create mode 100644 elastalert/alerters/alertmanager.py create mode 100644 tests/alerters/alertmanager_test.py diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 91546c50..a7868f76 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -28,6 +28,7 @@ Several rule types with common monitoring paradigms are included with ElastAlert Currently, we have support built in for these alert types: - Alerta +- Alertmanager - AWS SES (Amazon Simple Email Service) - AWS SNS (Amazon Simple Notification Service) - Chatwork diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 1f1fb883..4b7b50f1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1618,6 +1618,55 @@ Example usage using new-style format:: alerta_attributes_values: ["{key}", "{logdate}", "{sender_ip}" ] alerta_text: "Probe {hostname} is UP at {logdate} GMT" +Alertmanager +~~~~~~~~~~~~ + +This alert type will send alerts to Alertmanager postAlerts. ``alert_subject`` and ``alert_text`` are passed as the annotations labeled ``summary`` and ``description`` accordingly. The labels can be changed. +See https://prometheus.io/docs/alerting/clients/ for more details about the Alertmanager alert format. + +Required: + +``alertmanager_hosts``: The list of hosts pointing to the Alertmanager. + +Optional: + +``alertmanager_api_version``: Defaults to `v1`. Set to `v2` to enable the Alertmanager V2 API postAlerts. + +``alertmanager_alertname``: ``alertname`` is the only required label. Defaults to using the rule name of the alert. + +``alertmanager_labels``: Key:value pairs of arbitrary labels to be attached to every alert. Keys should match the regular expression ``^[a-zA-Z_][a-zA-Z0-9_]*$``. + +``alertmanager_annotations``: Key:value pairs of arbitrary annotations to be attached to every alert. Keys should match the regular expression ``^[a-zA-Z_][a-zA-Z0-9_]*$``. + +``alertmanager_fields``: Key:value pairs of labels and corresponding match fields. When using ``alertmanager_fields`` you can access nested fields and index into arrays the same way as with ``alert_text_args``. Keys should match the regular expression ``^[a-zA-Z_][a-zA-Z0-9_]*$``. This dictionary will be merged with the ``alertmanager_labels``. + +``alertmanager_alert_subject_labelname``: Rename the annotations' label name for ``alert_subject``. Default is ``summary``. + +``alertmanager_alert_text_labelname``: Rename the annotations' label name for ``alert_text``. Default is ``description``. + +``alertmanager_proxy``: By default ElastAlert 2 will not use a network proxy to send notifications to Alertmanager. Set this option using ``hostname:port`` if you need to use a proxy. only supports https. + +``alertmanager_ca_certs``: Set this option to ``True`` if you want to validate the SSL certificate. + +``alertmanager_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``False`` if you want to ignore SSL errors. + +``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles. + +Example usage:: + + alert: + - "alertmanager" + alertmanager_hosts: + - "http://alertmanager:9093" + alertmanager_alertname: "Title" + alertmanager_annotations: + severity: "error" + alertmanager_labels: + source: "elastalert" + alertmanager_fields: + msg: "message" + log: "@log_name" + AWS SES (Amazon Simple Email Service) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/elastalert/alerters/alertmanager.py b/elastalert/alerters/alertmanager.py new file mode 100644 index 00000000..573e4a72 --- /dev/null +++ b/elastalert/alerters/alertmanager.py @@ -0,0 +1,83 @@ +import json +import warnings + +import requests +from requests import RequestException + +from elastalert.alerts import Alerter, DateTimeEncoder +from elastalert.util import EAException, elastalert_logger, lookup_es_key + + +class AlertmanagerAlerter(Alerter): + """ Sends an alert to Alertmanager """ + + required_options = frozenset({'alertmanager_hosts'}) + + def __init__(self, rule): + super(AlertmanagerAlerter, self).__init__(rule) + self.api_version = self.rule.get('alertmanager_api_version', 'v1') + self.hosts = self.rule.get('alertmanager_hosts') + self.alertname = self.rule.get('alertmanager_alertname', self.rule.get('name')) + self.labels = self.rule.get('alertmanager_labels', dict()) + self.annotations = self.rule.get('alertmanager_annotations', dict()) + self.fields = self.rule.get('alertmanager_fields', dict()) + self.title_labelname = self.rule.get('alertmanager_alert_subject_labelname', 'summary') + self.body_labelname = self.rule.get('alertmanager_alert_text_labelname', 'description') + self.proxies =self.rule.get('alertmanager_proxy', None) + self.ca_certs = self.rule.get('alertmanager_ca_certs') + self.ignore_ssl_errors = self.rule.get('alertmanager_ignore_ssl_errors', False) + self.timeout = self.rule.get('alertmanager_timeout', 10) + + @staticmethod + def _json_or_string(obj): + """helper to encode non-string objects to JSON""" + if isinstance(obj, str): + return obj + return json.dumps(obj, cls=DateTimeEncoder) + + def alert(self, matches): + headers = {'content-type': 'application/json'} + proxies = {'https': self.proxies} if self.proxies else None + + self.labels.update({ + label: self._json_or_string(lookup_es_key(matches[0], term)) + for label, term in self.fields.items()}) + self.labels.update( + alertname=self.alertname, + elastalert_rule=self.rule.get('name')) + self.annotations.update({ + self.title_labelname: self.create_title(matches), + self.body_labelname: self.create_alert_body(matches)}) + payload = { + 'annotations': self.annotations, + 'labels': self.labels + } + + for host in self.hosts: + try: + url = '{}/api/{}/alerts'.format(host, self.api_version) + + if self.ca_certs: + verify = self.ca_certs + else: + verify = not self.ignore_ssl_errors + if self.ignore_ssl_errors: + requests.packages.urllib3.disable_warnings() + + response = requests.post( + url, + data=json.dumps([payload], cls=DateTimeEncoder), + headers=headers, + verify=verify, + proxies=proxies, + timeout=self.timeout + ) + + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Alertmanager: %s" % e) + elastalert_logger.info("Alert sent to Alertmanager") + + def get_info(self): + return {'type': 'alertmanager'} diff --git a/elastalert/loaders.py b/elastalert/loaders.py index bf35e60a..218c4080 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -36,6 +36,7 @@ from elastalert import alerts from elastalert import enhancements from elastalert import ruletypes +from elastalert.alerters.alertmanager import AlertmanagerAlerter from elastalert.alerters.email import EmailAlerter from elastalert.alerters.jira import JiraAlerter from elastalert.alerters.mattermost import MattermostAlerter @@ -88,6 +89,7 @@ class RulesLoader(object): # Used to map names of alerts to their classes alerts_mapping = { + 'alertmanager': AlertmanagerAlerter, 'tencent_sms': TencentSMSAlerter, 'email': EmailAlerter, 'jira': JiraAlerter, diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 0cee2848..2e18d459 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -285,6 +285,52 @@ properties: alerta_attributes_keys: {type: array, items: {type: string}} alerta_attributes_values: {type: array, items: {type: string}} # Python format string + ### Alertmanager + alertmanager_hosts: {type: array, items: {type: string}} + alertmanager_api_version: {type: string, enum: ['v1', 'v2']} + alertmanager_alert_subject_labelname: {type: string} + alertmanager_alert_text_labelname: {type: string} + alertmanager_proxy: {type: string} + alertmanager_ca_certs: {type: boolean} + alertmanager_ignore_ssl_errors: {type: boolean} + alertmanager_timeout: {type: integer} + alertmanager_labels: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + alertmanager_annotations: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + alertmanager_fields: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + ### AWS SES ses_email: *arrayOfString ses_from_addr: {type: string} diff --git a/tests/alerters/alertmanager_test.py b/tests/alerters/alertmanager_test.py new file mode 100644 index 00000000..f3ec3cd3 --- /dev/null +++ b/tests/alerters/alertmanager_test.py @@ -0,0 +1,324 @@ +import json +import logging +import pytest + +from unittest import mock + +from requests import RequestException + +from elastalert.alerters.alertmanager import AlertmanagerAlerter +from elastalert.loaders import FileRulesLoader +from elastalert.util import EAException + + +def test_alertmanager(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, "Alert sent to Alertmanager") == caplog.record_tuples[0] + + +def test_alertmanager_porxy(): + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alertmanager_proxy': 'http://proxy.url', + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies={'https': 'http://proxy.url'}, + verify=True, + timeout=10 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_alertmanager_timeout(): + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alertmanager_timeout': 20, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=20 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +@pytest.mark.parametrize('ca_certs, ignore_ssl_errors, excpet_verify', [ + ('', '', True), + ('', True, False), + ('', False, True), + (True, '', True), + (True, True, True), + (True, False, True), + (False, '', True), + (False, True, False), + (False, False, True) +]) +def test_alertmanager_ca_certs(ca_certs, ignore_ssl_errors, excpet_verify): + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alertmanager_alertname': 'Title', + 'alertmanager_annotations': {'severity': 'error'}, + 'alertmanager_labels': {'source': 'elastalert'}, + 'alertmanager_fields': {'msg': 'message', 'log': '@log_name'}, + 'alert_subject_args': ['message', '@log_name'], + 'alert': [] + } + if ca_certs: + rule['alertmanager_ca_certs'] = ca_certs + + if ignore_ssl_errors: + rule['alertmanager_ignore_ssl_errors'] = ignore_ssl_errors + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'message': 'Quit 123', + '@log_name': 'mysqld.general' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = [ + { + 'annotations': + { + 'severity': 'error', + 'summary': 'Test Alertmanager Rule', + 'description': 'Test Alertmanager Rule\n\n' + + '@log_name: mysqld.general\n' + + '@timestamp: 2021-01-01T00:00:00\n' + + 'message: Quit 123\nsomefield: foobarbaz\n' + }, + 'labels': { + 'source': 'elastalert', + 'msg': 'Quit 123', + 'log': 'mysqld.general', + 'alertname': 'Title', + 'elastalert_rule': 'Test Alertmanager Rule' + } + } + ] + + mock_post_request.assert_called_once_with( + 'http://alertmanager:9093/api/v1/alerts', + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=excpet_verify, + timeout=10 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_alertmanager_ea_exception(): + with pytest.raises(EAException) as ea: + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': ['http://alertmanager:9093'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + mock_run = mock.MagicMock(side_effect=RequestException) + with mock.patch('requests.post', mock_run), pytest.raises(RequestException): + alert.alert([match]) + assert 'Error posting to Alertmanager' in str(ea) + + +def test_alertmanager_getinfo(): + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alertmanager_hosts': 'http://alertmanager:9093', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + + expected_data = { + 'type': 'alertmanager' + } + actual_data = alert.get_info() + assert expected_data == actual_data + + +@pytest.mark.parametrize('alertmanager_hosts, expected_data', [ + ([], 'Missing required option(s): alertmanager_hosts'), + (['http://alertmanager:9093'], + { + 'type': 'alertmanager' + }), +]) +def test_alertmanager_required_error(alertmanager_hosts, expected_data): + try: + rule = { + 'name': 'Test Alertmanager Rule', + 'type': 'any', + 'alert': [] + } + + if alertmanager_hosts: + rule['alertmanager_hosts'] = alertmanager_hosts + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = AlertmanagerAlerter(rule) + + actual_data = alert.get_info() + assert expected_data == actual_data + except Exception as ea: + print('ea %s' % str(ea)) + assert expected_data in str(ea) From 2af0db06859a04720a523d58f3472fb9dd519a75 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 9 Oct 2021 00:23:59 +0900 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df15c48..f3fd6da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ - None ## New features -- None +- [Alertmanager] Added support for Alertmanager - [#503](https://github.com/jertel/elastalert2/pull/503) - @nsano-rururu ## Other changes - [Docs] Add exposed metrics documentation - [#498](https://github.com/jertel/elastalert2/pull/498) - @thisisxgp