Skip to content

Commit

Permalink
Merge pull request #823 from lepouletsuisse/fix_httppost2_nested_dict…
Browse files Browse the repository at this point in the history 
…ionnary

Fix HTTP post 2 rule for nested keys + safeguard for non-string headers
  • Loading branch information
@jertel
jertel authored Apr 27, 2022
2 parents 4c0afcb + 84b7bb0 commit 27ac39c
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 34 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
- Update documentation on Cloud ID support - [#810](https://github.com/jertel/elastalert2/pull/810) - @ferozsalam
- Upgrade tox 3.24.5 to 3.25.0 - [#813](https://github.com/jertel/elastalert2/pull/813) - @nsano-rururu
- [Kubernetes] Add support to specify rules directory - [#816](https://github.com/jertel/elastalert2/pull/816) @SBe
- Fix HTTP POST 2 alerter for nested payload keys - [#823](https://github.com/jertel/elastalert2/pull/823) - @lepouletsuisse

# 2.4.0

Expand Down
2 changes: 1 addition & 1 deletion docs/source/ruletypes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2329,7 +2329,7 @@ Required:

Optional:

``http_post2_payload``: List of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested fields). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"`
``http_post2_payload``: List of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested ES fields and nested payload keys). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"`.

``http_post2_raw_fields``: List of keys:values to use as the content of the POST. Example - ip:clientip will map the value from the clientip field of Elasticsearch to JSON key named ip. This field overwrite the keys with the same name in `http_post2_payload`.

Expand Down
33 changes: 16 additions & 17 deletions elastalert/alerters/httppost2.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import json

import requests
from jinja2 import Template
from requests import RequestException

from elastalert.alerts import Alerter, DateTimeEncoder
from elastalert.util import lookup_es_key, EAException, elastalert_logger
from jinja2 import Template


class HTTPPost2Alerter(Alerter):
Expand All @@ -31,33 +31,32 @@ def alert(self, matches):
""" Each match will trigger a POST to the specified endpoint(s). """
for match in matches:
payload = match if self.post_all_values else {}
for post_key, post_value in list(self.post_payload.items()):
post_key_template = Template(post_key)
post_key_res = post_key_template.render(**match)
post_value_template = Template(post_value)
post_value_res = post_value_template.render(**match)
payload[post_key_res] = post_value_res
payload_template = Template(json.dumps(self.post_payload))
payload_res = json.loads(payload_template.render(**match))
payload = {**payload, **payload_res}

for post_key, es_key in list(self.post_raw_fields.items()):
payload[post_key] = lookup_es_key(match, es_key)

headers = {
"Content-Type": "application/json",
"Accept": "application/json;charset=utf-8"
}
if self.post_ca_certs:
verify = self.post_ca_certs
else:
verify = not self.post_ignore_ssl_errors
if self.post_ignore_ssl_errors:
requests.packages.urllib3.disable_warnings()

for header_key, header_value in list(self.post_http_headers.items()):
header_key_template = Template(header_key)
header_key_res = header_key_template.render(**match)
header_value_template = Template(header_value)
header_value_res = header_value_template.render(**match)
headers[header_key_res] = header_value_res
header_template = Template(json.dumps(self.post_http_headers))
header_res = json.loads(header_template.render(**match))
headers = {
"Content-Type": "application/json",
"Accept": "application/json;charset=utf-8",
**header_res
}

for key, value in headers.items():
if type(value) in [type(None), list, dict]:
raise ValueError(f"HTTP Post 2: Can't send a header value which is not a string! "
f"Forbidden header {key}: {value}")

proxies = {'https': self.post_proxy} if self.post_proxy else None
for url in self.post_url:
Expand Down
170 changes: 154 additions & 16 deletions tests/alerters/httppost2_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import json
import logging
import pytest

from unittest import mock

import pytest
from requests import RequestException

from elastalert.alerters.httppost2 import HTTPPost2Alerter
Expand Down Expand Up @@ -182,6 +181,72 @@ def test_http_alerter_with_payload_args_keys(caplog):
assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0]


def test_http_alerter_with_payload_nested_keys(caplog):
caplog.set_level(logging.INFO)
rule = {
'name': 'Test HTTP Post Alerter With Payload args for the key',
'type': 'any',
'http_post2_url': 'http://test.webhook.url',
'http_post2_payload': {'key': {'nested_key': 'some_value_{{some_field}}'}},
'alert': []
}
rules_loader = FileRulesLoader({})
rules_loader.load_modules(rule)
alert = HTTPPost2Alerter(rule)
match = {
'@timestamp': '2017-01-01T00:00:00',
'some_field': 'toto'
}
with mock.patch('requests.post') as mock_post_request:
alert.alert([match])
expected_data = {
'key': {'nested_key': 'some_value_toto'},
}
mock_post_request.assert_called_once_with(
rule['http_post2_url'],
data=mock.ANY,
headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'},
proxies=None,
timeout=10,
verify=True
)
assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])
assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0]


def test_http_alerter_with_payload_none_value(caplog):
caplog.set_level(logging.INFO)
rule = {
'name': 'Test HTTP Post Alerter With Payload args for the key',
'type': 'any',
'http_post2_url': 'http://test.webhook.url',
'http_post2_payload': {'key': None},
'alert': []
}
rules_loader = FileRulesLoader({})
rules_loader.load_modules(rule)
alert = HTTPPost2Alerter(rule)
match = {
'@timestamp': '2017-01-01T00:00:00',
'some_field': 'toto'
}
with mock.patch('requests.post') as mock_post_request:
alert.alert([match])
expected_data = {
'key': None,
}
mock_post_request.assert_called_once_with(
rule['http_post2_url'],
data=mock.ANY,
headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'},
proxies=None,
timeout=10,
verify=True
)
assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])
assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0]


def test_http_alerter_with_payload_args_key_not_found(caplog):
caplog.set_level(logging.INFO)
rule = {
Expand Down Expand Up @@ -353,6 +418,78 @@ def test_http_alerter_with_header_args_value(caplog):
assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0]


def test_http_alerter_with_header_args_value_list(caplog):
with pytest.raises(ValueError) as error:
rule = {
'name': 'Test HTTP Post Alerter With Headers args value',
'type': 'any',
'http_post2_url': 'http://test.webhook.url',
'http_post2_headers': {'header_name': ["test1", "test2"]},
'http_post2_payload': {'posted_name': 'toto'},
'alert': []
}
rules_loader = FileRulesLoader({})
rules_loader.load_modules(rule)
alert = HTTPPost2Alerter(rule)
match = {
'@timestamp': '2017-01-01T00:00:00',
'titi': 'foobarbaz'
}
with mock.patch('requests.post'):
alert.alert([match])

assert "HTTP Post 2: Can't send a header value which is not a string! " \
"Forbidden header header_name: ['test1', 'test2']" in str(error)


def test_http_alerter_with_header_args_value_dict(caplog):
with pytest.raises(ValueError) as error:
rule = {
'name': 'Test HTTP Post Alerter With Headers args value',
'type': 'any',
'http_post2_url': 'http://test.webhook.url',
'http_post2_headers': {'header_name': {'test': 'val'}},
'http_post2_payload': {'posted_name': 'toto'},
'alert': []
}
rules_loader = FileRulesLoader({})
rules_loader.load_modules(rule)
alert = HTTPPost2Alerter(rule)
match = {
'@timestamp': '2017-01-01T00:00:00',
'titi': 'foobarbaz'
}
with mock.patch('requests.post'):
alert.alert([match])

assert "HTTP Post 2: Can't send a header value which is not a string! " \
"Forbidden header header_name: {'test': 'val'}" in str(error)


def test_http_alerter_with_header_args_value_none(caplog):
with pytest.raises(ValueError) as error:
rule = {
'name': 'Test HTTP Post Alerter With Headers args value',
'type': 'any',
'http_post2_url': 'http://test.webhook.url',
'http_post2_headers': {'header_name': None},
'http_post2_payload': {'posted_name': 'toto'},
'alert': []
}
rules_loader = FileRulesLoader({})
rules_loader.load_modules(rule)
alert = HTTPPost2Alerter(rule)
match = {
'@timestamp': '2017-01-01T00:00:00',
'titi': 'foobarbaz'
}
with mock.patch('requests.post'):
alert.alert([match])

assert "HTTP Post 2: Can't send a header value which is not a string! " \
"Forbidden header header_name: None" in str(error)


def test_http_alerter_with_header_args_value_not_found(caplog):
caplog.set_level(logging.INFO)
rule = {
Expand Down Expand Up @@ -644,7 +781,8 @@ def test_http_alerter_headers():
mock_post_request.assert_called_once_with(
rule['http_post2_url'],
data=mock.ANY,
headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8', 'authorization': 'Basic 123dr3234'},
headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8',
'authorization': 'Basic 123dr3234'},
proxies=None,
timeout=10,
verify=True
Expand All @@ -653,14 +791,14 @@ def test_http_alerter_headers():


@pytest.mark.parametrize('ca_certs, ignore_ssl_errors, excpet_verify', [
('', '', True),
('', True, False),
('', False, True),
(True, '', True),
(True, True, True),
(True, False, True),
(False, '', True),
(False, True, False),
('', '', True),
('', True, False),
('', False, True),
(True, '', True),
(True, True, True),
(True, False, True),
(False, '', True),
(False, True, False),
(False, False, True)
])
def test_http_alerter_post_ca_certs(ca_certs, ignore_ssl_errors, excpet_verify):
Expand Down Expand Up @@ -742,12 +880,12 @@ def test_http_getinfo():


@pytest.mark.parametrize('http_post2_url, expected_data', [
('', 'Missing required option(s): http_post2_url'),
('', 'Missing required option(s): http_post2_url'),
('http://test.webhook.url',
{
'type': 'http_post2',
'http_post2_webhook_url': ['http://test.webhook.url']
}),
{
'type': 'http_post2',
'http_post2_webhook_url': ['http://test.webhook.url']
}),
])
def test_http_required_error(http_post2_url, expected_data):
try:
Expand Down

0 comments on commit 27ac39c

Please sign in to comment.