diff --git a/CHANGELOG.md b/CHANGELOG.md index ac5e92b9..5ed1b176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - Update documentation on Cloud ID support - [#810](https://github.com/jertel/elastalert2/pull/810) - @ferozsalam - Upgrade tox 3.24.5 to 3.25.0 - [#813](https://github.com/jertel/elastalert2/pull/813) - @nsano-rururu - [Kubernetes] Add support to specify rules directory - [#816](https://github.com/jertel/elastalert2/pull/816) @SBe +- Fix HTTP POST 2 alerter for nested payload keys - [#823](https://github.com/jertel/elastalert2/pull/823) - @lepouletsuisse # 2.4.0 diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index e8217fb8..d0134041 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2329,7 +2329,7 @@ Required: Optional: -``http_post2_payload``: List of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested fields). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"` +``http_post2_payload``: List of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested ES fields and nested payload keys). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"`. ``http_post2_raw_fields``: List of keys:values to use as the content of the POST. Example - ip:clientip will map the value from the clientip field of Elasticsearch to JSON key named ip. This field overwrite the keys with the same name in `http_post2_payload`. diff --git a/elastalert/alerters/httppost2.py b/elastalert/alerters/httppost2.py index 3f1073f8..5f4c1ebb 100644 --- a/elastalert/alerters/httppost2.py +++ b/elastalert/alerters/httppost2.py @@ -1,11 +1,11 @@ import json import requests +from jinja2 import Template from requests import RequestException from elastalert.alerts import Alerter, DateTimeEncoder from elastalert.util import lookup_es_key, EAException, elastalert_logger -from jinja2 import Template class HTTPPost2Alerter(Alerter): @@ -31,20 +31,13 @@ def alert(self, matches): """ Each match will trigger a POST to the specified endpoint(s). """ for match in matches: payload = match if self.post_all_values else {} - for post_key, post_value in list(self.post_payload.items()): - post_key_template = Template(post_key) - post_key_res = post_key_template.render(**match) - post_value_template = Template(post_value) - post_value_res = post_value_template.render(**match) - payload[post_key_res] = post_value_res + payload_template = Template(json.dumps(self.post_payload)) + payload_res = json.loads(payload_template.render(**match)) + payload = {**payload, **payload_res} for post_key, es_key in list(self.post_raw_fields.items()): payload[post_key] = lookup_es_key(match, es_key) - headers = { - "Content-Type": "application/json", - "Accept": "application/json;charset=utf-8" - } if self.post_ca_certs: verify = self.post_ca_certs else: @@ -52,12 +45,18 @@ def alert(self, matches): if self.post_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() - for header_key, header_value in list(self.post_http_headers.items()): - header_key_template = Template(header_key) - header_key_res = header_key_template.render(**match) - header_value_template = Template(header_value) - header_value_res = header_value_template.render(**match) - headers[header_key_res] = header_value_res + header_template = Template(json.dumps(self.post_http_headers)) + header_res = json.loads(header_template.render(**match)) + headers = { + "Content-Type": "application/json", + "Accept": "application/json;charset=utf-8", + **header_res + } + + for key, value in headers.items(): + if type(value) in [type(None), list, dict]: + raise ValueError(f"HTTP Post 2: Can't send a header value which is not a string! " + f"Forbidden header {key}: {value}") proxies = {'https': self.post_proxy} if self.post_proxy else None for url in self.post_url: diff --git a/tests/alerters/httppost2_test.py b/tests/alerters/httppost2_test.py index 2dce1305..dbddeb98 100644 --- a/tests/alerters/httppost2_test.py +++ b/tests/alerters/httppost2_test.py @@ -1,9 +1,8 @@ import json import logging -import pytest - from unittest import mock +import pytest from requests import RequestException from elastalert.alerters.httppost2 import HTTPPost2Alerter @@ -182,6 +181,72 @@ def test_http_alerter_with_payload_args_keys(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] +def test_http_alerter_with_payload_nested_keys(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the key', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'key': {'nested_key': 'some_value_{{some_field}}'}}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'key': {'nested_key': 'some_value_toto'}, + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_none_value(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the key', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'key': None}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'key': None, + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + def test_http_alerter_with_payload_args_key_not_found(caplog): caplog.set_level(logging.INFO) rule = { @@ -353,6 +418,78 @@ def test_http_alerter_with_header_args_value(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] +def test_http_alerter_with_header_args_value_list(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': ["test1", "test2"]}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: ['test1', 'test2']" in str(error) + + +def test_http_alerter_with_header_args_value_dict(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': {'test': 'val'}}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: {'test': 'val'}" in str(error) + + +def test_http_alerter_with_header_args_value_none(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': None}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: None" in str(error) + + def test_http_alerter_with_header_args_value_not_found(caplog): caplog.set_level(logging.INFO) rule = { @@ -644,7 +781,8 @@ def test_http_alerter_headers(): mock_post_request.assert_called_once_with( rule['http_post2_url'], data=mock.ANY, - headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8', 'authorization': 'Basic 123dr3234'}, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8', + 'authorization': 'Basic 123dr3234'}, proxies=None, timeout=10, verify=True @@ -653,14 +791,14 @@ def test_http_alerter_headers(): @pytest.mark.parametrize('ca_certs, ignore_ssl_errors, excpet_verify', [ - ('', '', True), - ('', True, False), - ('', False, True), - (True, '', True), - (True, True, True), - (True, False, True), - (False, '', True), - (False, True, False), + ('', '', True), + ('', True, False), + ('', False, True), + (True, '', True), + (True, True, True), + (True, False, True), + (False, '', True), + (False, True, False), (False, False, True) ]) def test_http_alerter_post_ca_certs(ca_certs, ignore_ssl_errors, excpet_verify): @@ -742,12 +880,12 @@ def test_http_getinfo(): @pytest.mark.parametrize('http_post2_url, expected_data', [ - ('', 'Missing required option(s): http_post2_url'), + ('', 'Missing required option(s): http_post2_url'), ('http://test.webhook.url', - { - 'type': 'http_post2', - 'http_post2_webhook_url': ['http://test.webhook.url'] - }), + { + 'type': 'http_post2', + 'http_post2_webhook_url': ['http://test.webhook.url'] + }), ]) def test_http_required_error(http_post2_url, expected_data): try: