diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a6b0bc..b40841b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ ## New features - [Graylog GELF] Alerter added. [#1050](https://github.com/jertel/elastalert2/pull/1050) - @malinkinsa - [TheHive] Format `title`, `type`, and `source` with dynamic lookup values - [#1092](https://github.com/jertel/elastalert2/pull/1092) - @fandigunawan +- [HTTP POST2] `http_post2_payload` and `http_post2_headers` now support multiline JSON strings for better control over jinja templates - @akusei +- [HTTP POST2] This alerter now supports the use of `jinja_root_name` - @akusei +- [Rule Testing] The data file passed with `--data` can now contain a single JSON document or a list of JSON objects - @akusei ## Other changes - [Docs] Clarify Jira Cloud authentication configuration - [94f7e8c](https://github.com/jertel/elastalert2/commit/94f7e8cc98d59a00349e3b23acd8a8549c80dbc8) - @jertel @@ -16,7 +19,10 @@ - Modify schema to allow string and boolean for `*_ca_certs` to allow for one to specify a cert bundle for SSL certificate verification - [#1082](https://github.com/jertel/elastalert2/pull/1082) - @goggin - Fix UnicodeEncodeError in PagerDutyAlerter - [#1091](https://github.com/jertel/elastalert2/pull/1091) - @nsano-rururu - The scan_entire_timeframe setting, when used with use_count_query or use_terms_query will now scan entire timeframe on subsequent rule runs - [#1097](https://github.com/jertel/elastalert2/pull/1097) - @rschirin - +- Add new unit tests to cover changes in the HTTP POST2 alerter - @akusei +- [Docs] Updated HTTP POST2 documentation to outline changes with payloads, headers and multiline JSON strings - @akusei +- [HTTP POST2] Additional error checking around rendering and dumping payloads/headers to JSON - @akusei + # 2.9.0 ## Breaking changes diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 1b063265..224a7af3 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2445,11 +2445,11 @@ Required: Optional: -``http_post2_payload``: List of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested ES fields and nested payload keys). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"`. +``http_post2_payload``: A JSON string or list of keys:values to use for the payload of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested ES fields and nested payload keys). If not defined, all the Elasticsearch keys will be sent. Ex: `"description_{{ my_field }}": "Type: {{ type }}\\nSubject: {{ title }}"`. When field names use dot notation or reserved characters, _data can be used to access these fields. If _data conflicts with your top level data, use jinja_root_name to change its name. ``http_post2_raw_fields``: List of keys:values to use as the content of the POST. Example - ip:clientip will map the value from the clientip field of Elasticsearch to JSON key named ip. This field overwrite the keys with the same name in `http_post2_payload`. -``http_post2_headers``: List of keys:values to use for as headers of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested fields). Ex: `"Authorization": "{{ user }}"`. Headers `"Content-Type": "application/json"` and `"Accept": "application/json;charset=utf-8"` are present by default, you can overwrite them if you think this is necessary. +``http_post2_headers``: A JSON string or list of keys:values to use for as headers of the HTTP Post. You can use {{ field }} (Jinja2 template) in the key and the value to reference any field in the matched events (works for nested fields). Ex: `"Authorization": "{{ user }}"`. Headers `"Content-Type": "application/json"` and `"Accept": "application/json;charset=utf-8"` are present by default, you can overwrite them if you think this is necessary. When field names use dot notation or reserved characters, _data can be used to access these fields. If _data conflicts with your top level data, use jinja_root_name to change its name. ``http_post2_proxy``: URL of proxy, if required. only supports https. @@ -2461,6 +2461,32 @@ Optional: ``http_post2_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``True`` if you want to ignore SSL errors. +.. note:: Due to how values are rendered to JSON, the http_post2_headers and http_post2_payload fields require single quotes where quotes are required for Jinja templating. This only applies when using the YAML key:value pairs. Any quotes can be used with the new JSON string format. See below for examples of how to properly use quotes as well as an example of the new JSON string formatting. + +Incorrect usage with double quotes:: + + alert: post2 + http_post2_url: "http://example.com/api" + http_post2_payload: + # this will result in an error as " is escaped to \" + description: 'hello {{ _data["name"] }}' + # this will result in an error as " is escaped to \" + state: '{{ ["low","medium","high","critical"][event.severity] }}' + http_post2_headers: + authorization: Basic 123dr3234 + X-custom-type: '{{type}}' + +Correct usage with single quotes:: + + alert: post2 + http_post2_url: "http://example.com/api" + http_post2_payload: + description: hello {{ _data['name'] }} + state: "{{ ['low','medium','high','critical'][event.severity] }}" + http_post2_headers: + authorization: Basic 123dr3234 + X-custom-type: '{{type}}' + Example usage:: alert: post2 @@ -2474,6 +2500,27 @@ Example usage:: authorization: Basic 123dr3234 X-custom-type: {{type}} +Example usage with json string formatting:: + + alert: post2 + jinja_root_name: _new_root + http_post2_url: "http://example.com/api" + http_post2_payload: | + { + "description": "An event came from IP {{ _new_root["client.ip"] }}", + "username": "{{ _new_root['username'] }}" + {%- for k, v in some_field.items() -%} + ,"{{ k }}": "changed_{{ v }}" + {%- endfor -%} + } + http_post2_raw_fields: + ip: clientip + http_post2_headers: | + { + "authorization": "Basic 123dr3234", + "X-custom-{{key}}": "{{type}}" + } + Jira ~~~~ diff --git a/elastalert/alerters/httppost2.py b/elastalert/alerters/httppost2.py index f6537272..0980c061 100644 --- a/elastalert/alerters/httppost2.py +++ b/elastalert/alerters/httppost2.py @@ -1,15 +1,18 @@ import json +from json import JSONDecodeError import requests -from jinja2 import Template +from jinja2 import Template, TemplateSyntaxError from requests import RequestException from elastalert.alerts import Alerter, DateTimeEncoder from elastalert.util import lookup_es_key, EAException, elastalert_logger + def _json_escape(s): return json.encoder.encode_basestring(s)[1:-1] + def _escape_all_values(x): """recursively rebuilds, and escapes all strings for json, the given dict/list""" if isinstance(x, dict): @@ -21,6 +24,14 @@ def _escape_all_values(x): return x +def _render_json_template(template, match): + if not isinstance(template, str): + template = json.dumps(template) + template = Template(template) + + return json.loads(template.render(**match)) + + class HTTPPost2Alerter(Alerter): """ Requested elasticsearch indices are sent by HTTP POST. Encoded with JSON. """ required_options = frozenset(['http_post2_url']) @@ -39,15 +50,40 @@ def __init__(self, rule): self.post_ca_certs = self.rule.get('http_post2_ca_certs') self.post_ignore_ssl_errors = self.rule.get('http_post2_ignore_ssl_errors', False) self.timeout = self.rule.get('http_post2_timeout', 10) + self.jinja_root_name = self.rule.get('jinja_root_name', None) def alert(self, matches): """ Each match will trigger a POST to the specified endpoint(s). """ for match in matches: match_js_esc = _escape_all_values(match) - payload = match if self.post_all_values else {} - payload_template = Template(json.dumps(self.post_payload)) - payload_res = json.loads(payload_template.render(**match_js_esc)) - payload = {**payload, **payload_res} + args = {**match_js_esc} + if self.jinja_root_name: + args[self.jinja_root_name] = match_js_esc + + try: + field = 'payload' + payload = match if self.post_all_values else {} + payload_res = _render_json_template(self.post_payload, args) + payload = {**payload, **payload_res} + + field = 'headers' + header_res = _render_json_template(self.post_http_headers, args) + headers = { + "Content-Type": "application/json", + "Accept": "application/json;charset=utf-8", + **header_res + } + except TemplateSyntaxError as e: + raise ValueError(f"HTTP Post 2: The value of 'http_post2_{field}' has an invalid Jinja2 syntax. " + f"Please check your template syntax: {e}") + + except JSONDecodeError as e: + raise ValueError(f"HTTP Post 2: The rendered value for 'http_post2_{field}' contains invalid JSON. " + f"Please check your template syntax: {e}") + + except Exception as e: + raise ValueError(f"HTTP Post 2: An unexpected error occurred with the 'http_post2_{field}' value. " + f"Please check your template syntax: {e}") for post_key, es_key in list(self.post_raw_fields.items()): payload[post_key] = lookup_es_key(match, es_key) @@ -59,14 +95,6 @@ def alert(self, matches): if self.post_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() - header_template = Template(json.dumps(self.post_http_headers)) - header_res = json.loads(header_template.render(**match_js_esc)) - headers = { - "Content-Type": "application/json", - "Accept": "application/json;charset=utf-8", - **header_res - } - for key, value in headers.items(): if type(value) in [type(None), list, dict]: raise ValueError(f"HTTP Post 2: Can't send a header value which is not a string! " diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 2a27d380..b738bd50 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -14,7 +14,6 @@ from elastalert.config import load_conf from elastalert.elastalert import ElastAlerter -from elastalert.util import EAException from elastalert.util import elasticsearch_client from elastalert.util import lookup_es_key from elastalert.util import ts_now @@ -354,6 +353,8 @@ def run_elastalert(self, rule, conf): if not self.data: return None try: + if isinstance(self.data, dict): + self.data = [self.data] self.data.sort(key=lambda x: x[timestamp_field]) self.starttime = self.str_to_ts(self.data[0][timestamp_field]) self.endtime = self.str_to_ts(self.data[-1][timestamp_field]) + datetime.timedelta(seconds=1) diff --git a/tests/alerters/httppost2_test.py b/tests/alerters/httppost2_test.py index 5a2ca7e9..eae8fa25 100644 --- a/tests/alerters/httppost2_test.py +++ b/tests/alerters/httppost2_test.py @@ -3,6 +3,7 @@ from unittest import mock import pytest +import yaml from requests import RequestException from elastalert.alerters.httppost2 import HTTPPost2Alerter @@ -43,17 +44,20 @@ def test_http_alerter_with_payload(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_raw_fields(caplog): +def test_http_alerter_with_payload_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Payload and raw fields', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'posted_name': 'toto'}, - 'http_post2_raw_fields': {'posted_raw_field': 'somefield'}, - 'http_post2_static_payload': {'name': 'somestaticname'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload as JSON string + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "posted_name": "toto" + } + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) @@ -65,7 +69,6 @@ def test_http_alerter_with_payload_raw_fields(caplog): alert.alert([match]) expected_data = { 'posted_name': 'toto', - 'posted_raw_field': 'foobarbaz' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -79,14 +82,14 @@ def test_http_alerter_with_payload_raw_fields(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_raw_fields_overwrite(caplog): +def test_http_alerter_with_payload_raw_fields(caplog): caplog.set_level(logging.INFO) rule = { - 'name': 'Test HTTP Post Alerter raw fields overwrite payload', + 'name': 'Test HTTP Post Alerter With Payload and raw fields', 'type': 'any', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'posted_name': 'toto', 'overwrite_field': 'tata'}, - 'http_post2_raw_fields': {'overwrite_field': 'somefield'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'http_post2_raw_fields': {'posted_raw_field': 'somefield'}, 'http_post2_static_payload': {'name': 'somestaticname'}, 'alert': [] } @@ -101,7 +104,7 @@ def test_http_alerter_with_payload_raw_fields_overwrite(caplog): alert.alert([match]) expected_data = { 'posted_name': 'toto', - 'overwrite_field': 'foobarbaz' + 'posted_raw_field': 'foobarbaz' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -115,26 +118,36 @@ def test_http_alerter_with_payload_raw_fields_overwrite(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_no_clash(caplog): +def test_http_alerter_with_payload_raw_fields_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Payload has no clash with the match fields', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload and raw fields as JSON string + type: any + http_post2_url: http://test.webhook.url + http_post2_raw_fields: + posted_raw_field: somefield + http_post2_static_payload: + name: somestaticname + http_post2_payload: | + { + "posted_name": "toto" + } + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'toto': 'foobarbaz' + 'somefield': 'foobarbaz' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { 'posted_name': 'toto', + 'posted_raw_field': 'foobarbaz' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -148,13 +161,15 @@ def test_http_alerter_with_payload_no_clash(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_args_keys(caplog): +def test_http_alerter_with_payload_raw_fields_overwrite(caplog): caplog.set_level(logging.INFO) rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the key', + 'name': 'Test HTTP Post Alerter raw fields overwrite payload', 'type': 'any', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'args_{{some_field}}': 'tata'}, + 'http_post2_payload': {'posted_name': 'toto', 'overwrite_field': 'tata'}, + 'http_post2_raw_fields': {'overwrite_field': 'somefield'}, + 'http_post2_static_payload': {'name': 'somestaticname'}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -162,12 +177,13 @@ def test_http_alerter_with_payload_args_keys(caplog): alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'to\tto' # include some specially handled control char + 'somefield': 'foobarbaz' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { - 'args_to\tto': 'tata', + 'posted_name': 'toto', + 'overwrite_field': 'foobarbaz' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -181,59 +197,37 @@ def test_http_alerter_with_payload_args_keys(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_nested_keys(caplog): +def test_http_alerter_with_payload_raw_fields_overwrite_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the key', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'key': {'nested_key': 'some_value_{{some_field}}'}}, - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HTTPPost2Alerter(rule) - match = { - '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'toto' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - expected_data = { - 'key': {'nested_key': 'some_value_toto'}, - } - mock_post_request.assert_called_once_with( - rule['http_post2_url'], - data=mock.ANY, - headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, - proxies=None, - timeout=10, - verify=True + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter raw fields overwrite payload as a JSON string + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "posted_name": "toto", + "overwrite_field": "tata" + } + http_post2_raw_fields: + overwrite_field: somefield + http_post2_static_payload: + name: somestaticname + alert: [] + ''' ) - assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) - assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] - - -def test_http_alerter_with_payload_none_value(caplog): - caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the key', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'key': None}, - 'alert': [] - } rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'toto' + 'somefield': 'foobarbaz' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { - 'key': None, + 'posted_name': 'toto', + 'overwrite_field': 'foobarbaz' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -247,13 +241,13 @@ def test_http_alerter_with_payload_none_value(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_args_key_not_found(caplog): +def test_http_alerter_with_payload_no_clash(caplog): caplog.set_level(logging.INFO) rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the key if not found', + 'name': 'Test HTTP Post Alerter With Payload has no clash with the match fields', 'type': 'any', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'args_{{some_field1}}': 'tata'}, + 'http_post2_payload': {'posted_name': 'toto'}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -261,12 +255,12 @@ def test_http_alerter_with_payload_args_key_not_found(caplog): alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'toto' + 'toto': 'foobarbaz' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { - 'args_': 'tata', + 'posted_name': 'toto', } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -280,27 +274,29 @@ def test_http_alerter_with_payload_args_key_not_found(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_args_value(caplog): +def test_http_alerter_with_payload_no_clash_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the value', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'posted_name': 'toto', 'args_name': '{{some_field}}'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) has no clash with the match fields + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"posted_name": "toto"} + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'foo\tbar\nbaz' # include some specially handled control chars + 'toto': 'foobarbaz' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { 'posted_name': 'toto', - 'args_name': 'foo\tbar\nbaz', } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -314,13 +310,13 @@ def test_http_alerter_with_payload_args_value(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_payload_args_value_not_found(caplog): +def test_http_alerter_with_payload_args_keys(caplog): caplog.set_level(logging.INFO) rule = { - 'name': 'Test HTTP Post Alerter With Payload args for the value if not found', + 'name': 'Test HTTP Post Alerter With Payload args for the key', 'type': 'any', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_payload': {'posted_name': 'toto', 'args_name': '{{some_field1}}'}, + 'http_post2_payload': {'args_{{some_field}}': 'tata'}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -328,13 +324,12 @@ def test_http_alerter_with_payload_args_value_not_found(caplog): alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'some_field': 'foobarbaz' + 'some_field': 'to\tto' # include some specially handled control char } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) expected_data = { - 'posted_name': 'toto', - 'args_name': '', + 'args_to\tto': 'tata', } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -348,84 +343,51 @@ def test_http_alerter_with_payload_args_value_not_found(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_header_no_clash(caplog): +def test_http_alerter_with_payload_args_keys_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Headers has no clash with the match fields', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': 'titi'}, - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HTTPPost2Alerter(rule) - match = { - '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - expected_headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json;charset=utf-8', - 'header_name': 'titi' - } - mock_post_request.assert_called_once_with( - rule['http_post2_url'], - data=mock.ANY, - headers=expected_headers, - proxies=None, - timeout=10, - verify=True + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the key + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"args_{{some_field}}": "tata"} + alert: [] + ''' ) - assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] - - -def test_http_alerter_with_header_args_value(caplog): - caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Headers args value', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': '{{titi}}'}, - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foo\tbarbaz' # include some specially handled control chars + 'some_field': 'to\tto' # include some specially handled control char } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) - expected_headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json;charset=utf-8', - 'header_name': 'foo\tbarbaz' + expected_data = { + 'args_to\tto': 'tata', } mock_post_request.assert_called_once_with( rule['http_post2_url'], data=mock.ANY, - headers=expected_headers, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, timeout=10, verify=True ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_header_args_value_list(caplog): +def test_http_alerter_with_payload_template_error(caplog): with pytest.raises(ValueError) as error: rule = { - 'name': 'Test HTTP Post Alerter With Headers args value', + 'name': 'Test HTTP Post Alerter With unexpected template syntax error', 'type': 'any', + 'jinja_root_name': '_data', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': ["test1", "test2"]}, - 'http_post2_payload': {'posted_name': 'toto'}, + 'http_post2_headers': {'header_name': 'toto'}, + 'http_post2_payload': {'posted_name': '{{ _data["titi"] }}'}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -438,18 +400,18 @@ def test_http_alerter_with_header_args_value_list(caplog): with mock.patch('requests.post'): alert.alert([match]) - assert "HTTP Post 2: Can't send a header value which is not a string! " \ - "Forbidden header header_name: ['test1', 'test2']" in str(error) + assert "HTTP Post 2: The value of 'http_post2_payload' has an invalid Jinja2 syntax." in str(error) -def test_http_alerter_with_header_args_value_dict(caplog): +def test_http_alerter_with_payload_unexpected_error(caplog): with pytest.raises(ValueError) as error: rule = { - 'name': 'Test HTTP Post Alerter With Headers args value', + 'name': 'Test HTTP Post Alerter With unexpected error', 'type': 'any', + 'jinja_root_name': '_data', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': {'test': 'val'}}, - 'http_post2_payload': {'posted_name': 'toto'}, + 'http_post2_headers': {'header_name': 'toto'}, + 'http_post2_payload': {'posted_name': '{% for k,v in titi %}{% endfor %}'}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -457,25 +419,26 @@ def test_http_alerter_with_header_args_value_dict(caplog): alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foobarbaz' + 'titi': {'foobarbaz': 'tata'} } with mock.patch('requests.post'): alert.alert([match]) - assert "HTTP Post 2: Can't send a header value which is not a string! " \ - "Forbidden header header_name: {'test': 'val'}" in str(error) + assert "HTTP Post 2: An unexpected error occurred with the 'http_post2_payload' value." in str(error) -def test_http_alerter_with_header_args_value_none(caplog): +def test_http_alerter_with_payload_json_decode_error(caplog): with pytest.raises(ValueError) as error: - rule = { - 'name': 'Test HTTP Post Alerter With Headers args value', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': None}, - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With json decode error + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + this is invalid json + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) @@ -486,53 +449,58 @@ def test_http_alerter_with_header_args_value_none(caplog): with mock.patch('requests.post'): alert.alert([match]) - assert "HTTP Post 2: Can't send a header value which is not a string! " \ - "Forbidden header header_name: None" in str(error) + assert "HTTP Post 2: The rendered value for 'http_post2_payload' contains invalid JSON." in str(error) -def test_http_alerter_with_header_args_value_not_found(caplog): +def test_http_alerter_with_payload_args_keys_jinja_root(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Headers args value if not found', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_name': '{{titi1}}'}, - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload args for the key using custom jinja root + type: any + jinja_root_name: _data + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "args_{{_data['key1']}}": "tata", + "args_{{_data["key2"]}}": "toto" + } + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foobarbaz' + 'key1': 'ta\tta', # include some specially handled control char + 'key2': 'to\tto', } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) - expected_headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json;charset=utf-8', - 'header_name': '' + expected_data = { + 'args_to\tto': 'toto', + 'args_ta\tta': 'tata', } mock_post_request.assert_called_once_with( rule['http_post2_url'], data=mock.ANY, - headers=expected_headers, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, timeout=10, verify=True ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_header_args_key(caplog): +def test_http_alerter_with_payload_nested_keys(caplog): caplog.set_level(logging.INFO) rule = { - 'name': 'Test HTTP Post Alerter With Headers args key', + 'name': 'Test HTTP Post Alerter With Payload args for the key', 'type': 'any', 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_{{titi}}': 'tata'}, - 'http_post2_payload': {'posted_name': 'toto'}, + 'http_post2_payload': {'key': {'nested_key': 'some_value_{{some_field}}'}}, 'alert': [] } rules_loader = FileRulesLoader({}) @@ -540,49 +508,1104 @@ def test_http_alerter_with_header_args_key(caplog): alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foobarbaz' + 'some_field': 'toto' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) - expected_headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json;charset=utf-8', - 'header_foobarbaz': 'tata' + expected_data = { + 'key': {'nested_key': 'some_value_toto'}, } mock_post_request.assert_called_once_with( rule['http_post2_url'], data=mock.ANY, - headers=expected_headers, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, timeout=10, verify=True ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] -def test_http_alerter_with_header_args_key_not_found(caplog): +def test_http_alerter_with_payload_nested_keys_as_json_string(caplog): caplog.set_level(logging.INFO) - rule = { - 'name': 'Test HTTP Post Alerter With Headers args key if not found', - 'type': 'any', - 'http_post2_url': 'http://test.webhook.url', - 'http_post2_headers': {'header_{{titi1}}': 'tata'}, - 'http_post2_payload': {'posted_name': 'toto'}, - 'alert': [] - } + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the key + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "key": {"nested_key": "some_value_{{some_field}}"} + } + alert: [] + ''' + ) rules_loader = FileRulesLoader({}) rules_loader.load_modules(rule) alert = HTTPPost2Alerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', - 'titi': 'foobarbaz' + 'some_field': 'toto' } with mock.patch('requests.post') as mock_post_request: alert.alert([match]) - expected_headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json;charset=utf-8', - 'header_': 'tata' + expected_data = { + 'key': {'nested_key': 'some_value_toto'}, + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_none_value(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the key', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'key': None}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'key': None, + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_none_value_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the key + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"key": null} + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'key': None, + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_key_not_found(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the key if not found', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'args_{{some_field1}}': 'tata'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'args_': 'tata', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_key_not_found_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the key if not found + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"args_{{some_field1}}": "tata"} + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'toto' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'args_': 'tata', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'posted_name': 'toto', 'args_name': '{{some_field}}'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foo\tbar\nbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name': 'foo\tbar\nbaz', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + """ + name: Test HTTP Post Alerter With Payload as json string and args for the value + type: any + http_post2_url: 'http://test.webhook.url' + http_post2_payload: | + { + "posted_name": "toto", + "args_name": "{{some_field}}" + } + alert: [] + """ + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foo\tbar\nbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name': 'foo\tbar\nbaz', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value_jinja_root(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the value using custom jinja root', + 'jinja_root_name': '_data', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'posted_name': 'toto', 'args_name': "{{_data['some_field']}}"}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foo\tbar\nbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name': 'foo\tbar\nbaz', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value_jinja_root_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the value using custom jinja root + jinja_root_name: _data + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "posted_name": "toto", + "args_name1": "{{_data['some_field']}}", + "args_name2": "{{_data["some_field"]}}" + } + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foo\tbar\nbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name1': 'foo\tbar\nbaz', + 'args_name2': 'foo\tbar\nbaz', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value_not_found(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Payload args for the value if not found', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_payload': {'posted_name': 'toto', 'args_name': '{{some_field1}}'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name': '', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_payload_args_value_not_found_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) args for the value if not found + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + { + "posted_name": "toto", + "args_name": "{{some_field1}}" + } + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'some_field': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + 'args_name': '', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_no_clash(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers has no clash with the match fields', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': 'titi'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'titi' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_no_clash_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers has no clash with the match fields + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": "titi"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'titi' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_value(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': '{{titi}}'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foo\tbarbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'foo\tbarbaz' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_value_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": "{{titi}}"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foo\tbarbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'foo\tbarbaz' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_template_error(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args template error', + 'jinja_root_name': '_data', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': '{{ _data["titi"] }}'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: The value of 'http_post2_headers' has an invalid Jinja2 syntax." in str(error) + + +def test_http_alerter_with_header_json_decode_error(caplog): + with pytest.raises(ValueError) as error: + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args json decode error + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + this is invalid json + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: The rendered value for 'http_post2_headers' contains invalid JSON." in str(error) + + +def test_http_alerter_with_header_args_value_jinja_root(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value using custom jinja root', + 'jinja_root_name': '_data', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': "{{_data['titi']}}"}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foo\tbarbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'foo\tbarbaz' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_value_jinja_root_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value using custom jinja root + type: any + jinja_root_name: _data + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": "{{_data['titi']}}"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foo\tbarbaz' # include some specially handled control chars + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': 'foo\tbarbaz' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_value_list(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': ["test1", "test2"]}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: ['test1', 'test2']" in str(error) + + +def test_http_alerter_with_header_args_value_list_as_json_string(caplog): + with pytest.raises(ValueError) as error: + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": ["test1", "test2"]} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: ['test1', 'test2']" in str(error) + + +def test_http_alerter_with_header_args_value_dict(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': {'test': 'val'}}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: {'test': 'val'}" in str(error) + + +def test_http_alerter_with_header_args_value_dict_as_json_string(caplog): + with pytest.raises(ValueError) as error: + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": {"test": "val"}} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: {'test': 'val'}" in str(error) + + +def test_http_alerter_with_header_args_value_none(caplog): + with pytest.raises(ValueError) as error: + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': None}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: None" in str(error) + + +def test_http_alerter_with_header_args_value_none_as_json_string(caplog): + with pytest.raises(ValueError) as error: + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": null} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post'): + alert.alert([match]) + + assert "HTTP Post 2: Can't send a header value which is not a string! " \ + "Forbidden header header_name: None" in str(error) + + +def test_http_alerter_with_header_args_value_not_found(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args value if not found', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_name': '{{titi1}}'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': '' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_value_not_found_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args value if not found as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_name": "{{titi1}}"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_name': '' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_key(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args key', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_{{titi}}': 'tata'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_foobarbaz': 'tata' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_key_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args key as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_{{titi}}": "tata"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_foobarbaz': 'tata' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_key_jinja_root(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args key using custom jinja root', + 'jinja_root_name': '_data', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {"header_{{_data['titi']}}": 'tata'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2023-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_foobarbaz': 'tata' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_key_not_found(caplog): + caplog.set_level(logging.INFO) + rule = { + 'name': 'Test HTTP Post Alerter With Headers args key if not found', + 'type': 'any', + 'http_post2_url': 'http://test.webhook.url', + 'http_post2_headers': {'header_{{titi1}}': 'tata'}, + 'http_post2_payload': {'posted_name': 'toto'}, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_': 'tata' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers=expected_headers, + proxies=None, + timeout=10, + verify=True + ) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + +def test_http_alerter_with_header_args_key_not_found_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Headers args key if not found as json string + type: any + http_post2_url: http://test.webhook.url + http_post2_headers: | + {"header_{{titi1}}": "tata"} + http_post2_payload: + posted_name: toto + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'titi': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8', + 'header_': 'tata' } mock_post_request.assert_called_once_with( rule['http_post2_url'], @@ -628,6 +1651,42 @@ def test_http_alerter_with_payload_nested(caplog): assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] +def test_http_alerter_with_payload_nested_as_json_string(caplog): + caplog.set_level(logging.INFO) + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"posted_name": "{{ toto.tata }}"} + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'toto': {'tata': 'titi'} + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'titi', + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert ('elastalert', logging.INFO, 'HTTP Post 2 alert sent.') == caplog.record_tuples[0] + + def test_http_alerter_with_payload_all_values(): rule = { 'name': 'Test HTTP Post Alerter With Payload', @@ -662,6 +1721,43 @@ def test_http_alerter_with_payload_all_values(): assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) +def test_http_alerter_with_payload_all_values_as_json_string(): + rule = yaml.safe_load( + ''' + name: Test HTTP Post Alerter With Payload (as JSON string) + type: any + http_post2_url: http://test.webhook.url + http_post2_payload: | + {"posted_name": "toto"} + http_post2_all_values: true + alert: [] + ''' + ) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = HTTPPost2Alerter(rule) + match = { + '@timestamp': '2017-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + 'posted_name': 'toto', + '@timestamp': '2017-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + mock_post_request.assert_called_once_with( + rule['http_post2_url'], + data=mock.ANY, + headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + def test_http_alerter_without_payload(): rule = { 'name': 'Test HTTP Post Alerter Without Payload',