From 45cda8dc427b12a7ec99301e41e6e670795e3f80 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 27 May 2023 23:43:32 +0900 Subject: [PATCH 01/24] Kibana Discover support kibana 8.8 --- docs/source/ruletypes.rst | 2 +- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 41 ++++++++++++++++++----------------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 80cc3486..c8fb449f 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -716,7 +716,7 @@ kibana_discover_version The currently supported versions of Kibana Discover are: - `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11`, `7.12`, `7.13`, `7.14`, `7.15`, `7.16`, `7.17` -- `8.0`, `8.1`, `8.2`, `8.3`, `8.4`, `8.5`, `8.6`, `8.7` +- `8.0`, `8.1`, `8.2`, `8.3`, `8.4`, `8.5`, `8.6`, `8.7`, `8.8` ``kibana_discover_version: '7.15'`` diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 892ff526..f2c9bb94 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -16,7 +16,7 @@ kibana_versions = frozenset([ '7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', - '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', '8.6', '8.7' + '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', '8.6', '8.7', '8.8' ]) def generate_kibana_discover_url(rule, match): diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index f3f46051..dd721bc9 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -280,7 +280,7 @@ properties: generate_kibana_discover_url: {type: boolean} shorten_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string} - kibana_discover_version: {type: string, enum: ['8.7', '8.6', '8.5', '8.4', '8.3', '8.2', '8.1', '8.0', '7.17', '7.16', '7.15', '7.14', '7.13', '7.12', '7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0']} + kibana_discover_version: {type: string, enum: ['8.8', '8.7', '8.6', '8.5', '8.4', '8.3', '8.2', '8.1', '8.0', '7.17', '7.16', '7.15', '7.14', '7.13', '7.12', '7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index 5e7a7182..202d14b6 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -30,7 +30,8 @@ '8.4', '8.5', '8.6', - '8.7' + '8.7', + '8.8' ]) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( @@ -68,7 +69,7 @@ def test_generate_kibana_discover_url_with_relative_kibana_discover_app_url(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'app/discover#/', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': '620ad0e6-43df-4557-bda2-384960fa9086', 'timestamp_field': 'timestamp' }, @@ -114,7 +115,7 @@ def test_generate_kibana_discover_url_with_missing_kibana_discover_version(): def test_generate_kibana_discover_url_with_missing_kibana_discover_app_url(): url = generate_kibana_discover_url( rule={ - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs', 'timestamp_field': 'timestamp', 'name': 'test' @@ -130,7 +131,7 @@ def test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'timestamp_field': 'timestamp', 'name': 'test' }, @@ -164,7 +165,7 @@ def test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitut url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' }, @@ -196,7 +197,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -229,7 +230,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -263,7 +264,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -296,7 +297,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -330,7 +331,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' @@ -363,7 +364,7 @@ def test_generate_kibana_discover_url_with_custom_columns(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'kibana_discover_columns': ['level', 'message'], 'timestamp_field': 'timestamp' @@ -396,7 +397,7 @@ def test_generate_kibana_discover_url_with_single_filter(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'filter': [ @@ -447,7 +448,7 @@ def test_generate_kibana_discover_url_with_multiple_filters(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', 'timestamp_field': 'timestamp', 'filter': [ @@ -501,7 +502,7 @@ def test_generate_kibana_discover_url_with_int_query_key(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'query_key': 'geo.dest' @@ -561,7 +562,7 @@ def test_generate_kibana_discover_url_with_str_query_key(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'query_key': 'geo.dest' @@ -623,7 +624,7 @@ def test_generate_kibana_discover_url_with_null_query_key_value(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'query_key': 'status' @@ -673,7 +674,7 @@ def test_generate_kibana_discover_url_with_missing_query_key_value(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'query_key': 'status' @@ -722,7 +723,7 @@ def test_generate_kibana_discover_url_with_compound_query_key(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'compound_query_key': ['geo.src', 'geo.dest'], @@ -810,7 +811,7 @@ def test_generate_kibana_discover_url_with_filter_and_query_key(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'filter': [ @@ -887,7 +888,7 @@ def test_generate_kibana_discover_url_with_querystring_filter_and_query_key(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '8.7', + 'kibana_discover_version': '8.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', 'filter': [ From 7ee8fe814c14298b95a9393e91fe8fdd98a24551 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 27 May 2023 23:47:07 +0900 Subject: [PATCH 02/24] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27024b13..66074c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - TBD ## Other changes -- TBD +- Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu # 2.11.0 From 433c521d18b0845a629b2d56b31540837b96e2c5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 4 Jun 2023 21:57:35 -0400 Subject: [PATCH 03/24] add support for EQL queries --- docs/source/recipes/writing_filters.rst | 26 +++++ elastalert/__init__.py | 83 +++++++++++++++- elastalert/eql.py | 51 ++++++++++ tests/eql_test.py | 127 ++++++++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 elastalert/eql.py create mode 100644 tests/eql_test.py diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index eb4da6b7..0548338a 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -125,3 +125,29 @@ Below is a more complex example for Elasticsearch 7.x, provided by a `community uid: charlie - match: menu_item: "burrito pasta salad pizza" + +EQL (Event Query Language) +************************** + +EQL is partially supported as of version 2.12.0. To use EQL, include a filter item as follows:: + + filter: + - eql: any where machine.os == "win 8" + +Note that only one ``eql`` filter can be defined in a filter. + +It is also possible to use standard query filters in combination with EQL filters:: + + filter: + - eql: any where machine.os == "win 8" + - query: + query_string: + query: "test.field: 123" + +EQL is only partially supported due to the following limitations: + +- Cannot be used with aggregation rule types. +- Cannot be used with blacklist/whitelist rule types. +- Cannot be used with percentage match rule types. +- Cannot be used with ``use_count_query`` property. +- Does not support scrolling, so large result sets may have unexpected results. Be sure to filter your queries thoroughly to avoid returning excessive numbers of events. diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 941ad857..66031a39 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import copy -import time +import elastalert.eql as eql from elasticsearch import Elasticsearch from elasticsearch import RequestsHttpConnection @@ -74,3 +74,84 @@ def resolve_writeback_index(self, writeback_index, doc_type): elif doc_type == 'elastalert_error': return writeback_index + '_error' return writeback_index + + + @query_params( + "_source", + "_source_excludes", + "_source_includes", + "allow_no_indices", + "allow_partial_search_results", + "analyze_wildcard", + "analyzer", + "batched_reduce_size", + "ccs_minimize_roundtrips", + "default_operator", + "df", + "docvalue_fields", + "expand_wildcards", + "explain", + "from_", + "ignore_throttled", + "ignore_unavailable", + "lenient", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "preference", + "q", + "request_cache", + "rest_total_hits_as_int", + "routing", + "scroll", + "search_type", + "seq_no_primary_term", + "size", + "sort", + "stats", + "stored_fields", + "suggest_field", + "suggest_mode", + "suggest_size", + "suggest_text", + "terminate_after", + "timeout", + "track_scores", + "track_total_hits", + "typed_keys", + "version", + ) + def search(self, body=None, index=None, doc_type=None, params=None, headers=None): + # This implementation of search is nearly identical to the base class with the following exceptions: + # 1. If the request body contains an EQL query, the body will be restructured to support the EQL API. + # 2. The path will be set to the EQL API endpoint, if #1 is true. + # 3. The scroll and _source_includes params will be dropped if #1 is true, since the EQL API doesn't support them. + # 4. The size param will be moved to a body parameter instead of a top-level param if #1 is true. + # 5. The results will be converted from EQL API format into the standard search format. + + # from is a reserved word so it cannot be used, use from_ instead + if "from_" in params: + params["from"] = params.pop("from_") + + path = _make_path(index, doc_type, "_search") + eql_body = eql.format_request(body) + if eql_body is not None: + path = path.replace('/_search', '/_eql/search') + body = eql_body + if 'size' in params: + body['size'] = int(params.pop('size')) + if 'scroll' in params: + params.pop('scroll') + if '_source_includes' in params: + params.pop('_source_includes') + + results = self.transport.perform_request( + "POST", + path, + params=params, + headers=headers, + body=body, + ) + + eql.format_results(results); + + return results diff --git a/elastalert/eql.py b/elastalert/eql.py new file mode 100644 index 00000000..9897dda2 --- /dev/null +++ b/elastalert/eql.py @@ -0,0 +1,51 @@ +# A dict containing EQL would resemble the following: +# {'query': {'bool': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'gt': 'yyyy...', 'lte': 'yyyy...'}}}, {'eql': 'process where process.name == "regsvr32.exe"'}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]} +def format_request(body): + query = body.get('query') + if not query: + return None + + query_bool = query.get('bool') + if not query_bool: + return None + + filter = query_bool.get('filter') + if not filter: + return None + + filter_bool = filter.get('bool') + if not filter_bool: + return None + + filter_bool_must = filter_bool.get('must') + if not filter_bool_must: + return None + + other_filters = [] + eql = None + for f in filter_bool_must: + if f.get('eql'): + eql = f['eql'] + else: + other_filters.append(f) + + if eql: + new_body = {'filter': { 'bool': { 'must': other_filters }}, 'query': eql} + return new_body + + return None + +def format_results(results): + hits = results.get('hits') + if not hits: + return results + + events = hits.get('events') + if events is None: + return results + + # relabel events as hits, for consistency + events = hits.pop('events') + hits['hits'] = events + + return results \ No newline at end of file diff --git a/tests/eql_test.py b/tests/eql_test.py new file mode 100644 index 00000000..f54ec6c7 --- /dev/null +++ b/tests/eql_test.py @@ -0,0 +1,127 @@ +import elastalert.eql as eql +from elastalert import ElasticSearchClient +from unittest import mock + + +def test_format_request_without_eql(): + assert eql.format_request({}) is None + assert eql.format_request({'query': {}}) is None + assert eql.format_request({'query': {'bool': {}}}) is None + assert eql.format_request({'query': {'bool': {'filter': {}}}}) is None + assert eql.format_request({'query': {'bool': {'filter': {'bool': {}}}}}) is None + assert eql.format_request({'query': {'bool': {'filter': {'bool': {'must': []}}}}}) is None + assert eql.format_request({'query': {'bool': {'filter': {'bool': {'must': [{'foo': 'bar'}]}}}}}) is None + + +def test_format_request_with_eql(): + body = { + 'query': { + 'bool': { + 'filter': { + 'bool': { + 'must': [ + {'eql': 'test query'}, + {'other': 'other filter'}, + ] + } + } + } + } + } + expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'test query'} + assert eql.format_request(body) == expected_body + + +def eql_body(): + body = { + 'query': { + 'bool': { + 'filter': { + 'bool': { + 'must': [ + {'eql': 'test query'}, + {'other': 'other filter'}, + {'eql': 'newer query'}, + ] + } + } + } + } + } + return body + + +def test_format_request_with_excessive_eql(): + body = eql_body() + expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'newer query'} + assert eql.format_request(body) == expected_body + + +def test_format_results_without_events(): + expected_results = {'hits': {'hits': []}} + results = expected_results + eql.format_results(results) == expected_results + + +def test_format_results_with_events(): + expected_results = {'hits': {'events': [{'foo': 'bar'}]}} + results = {'hits': {'hits': [{'foo': 'bar'}]}} + eql.format_results(results) == expected_results + + +def init_client(): + conn = { + 'es_host': '', + 'es_hosts': [], + 'es_port': 123, + 'es_url_prefix': '', + 'use_ssl': False, + 'verify_certs': False, + 'ca_certs': [], + 'ssl_show_warn': False, + 'http_auth': '', + 'headers': [], + 'es_conn_timeout': 0, + 'send_get_body_as': '', + 'client_cert': '', + 'client_key': '' + } + return ElasticSearchClient(conn) + + +def test_search_without_eql(): + es_client = init_client() + + expected_params = {'from': True, 'size': 12, 'scroll': True, '_source_includes': True} + expected_headers = {} + expected_body = {} + results = {} + es_client.transport = mock.Mock() + es_client.transport.perform_request.return_value = results + + body = {} + params = {'from_': True, 'size': 12, 'scroll': True, '_source_includes': True} + es_client.search(body=body, index='test', params=params) + es_client.transport.perform_request.assert_called_with('POST', '/test/_search', + params=expected_params, + headers=expected_headers, + body=expected_body) + + +def test_search_with_eql(): + es_client = init_client() + + expected_params = {'from': True} + expected_headers = {} + expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'newer query', 'size': 12} + results = {} + es_client.transport = mock.Mock() + es_client.transport.perform_request.return_value = results + + body = eql_body() + params = {'from_': True, 'size': 12, 'scroll': True, '_source_includes': True} + results = es_client.search(body=body, index='test', params=params) + es_client.transport.perform_request.assert_called_with('POST', '/test/_eql/search', + params=expected_params, + headers=expected_headers, + body=expected_body) From 3844ac77e66d925ac60e408f904b445d83f9635d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 4 Jun 2023 22:01:47 -0400 Subject: [PATCH 04/24] add support for EQL queries --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27024b13..9d453abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - TBD ## New features -- TBD +- Add initial support for EQL - [#1189](https://github.com/jertel/elastalert2/pull/1189) - @jertel ## Other changes - TBD From 3bf15506823d2df04eea990436958c7031c173c4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 4 Jun 2023 22:41:12 -0400 Subject: [PATCH 05/24] add support for EQL queries --- docs/source/recipes/writing_filters.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index 0548338a..54dd705c 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -151,3 +151,4 @@ EQL is only partially supported due to the following limitations: - Cannot be used with percentage match rule types. - Cannot be used with ``use_count_query`` property. - Does not support scrolling, so large result sets may have unexpected results. Be sure to filter your queries thoroughly to avoid returning excessive numbers of events. +- Not supported with OpenSearch \ No newline at end of file From cf8a0f173cb77b5f979db3465e4a9118a859736c Mon Sep 17 00:00:00 2001 From: Ryan Goggin Date: Wed, 7 Jun 2023 08:05:48 -0400 Subject: [PATCH 06/24] Add fields option to rules to allow for runtime fields and others --- docs/source/ruletypes.rst | 9 +++++++++ elastalert/elastalert.py | 3 +++ elastalert/loaders.py | 4 ++++ elastalert/schema.yaml | 1 + 4 files changed, 17 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c8fb449f..ac69e4e8 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -104,6 +104,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``include`` (list of strs, default ["*"]) | | +--------------------------------------------------------------+ | +| ``fields`` (list of strs, no default) | | ++--------------------------------------------------------------+ | | ``filter`` (ES filter DSL, no default) | | +--------------------------------------------------------------+ | | ``max_query_size`` (int, default global max_query_size) | | @@ -584,6 +586,13 @@ include fields, along with '@timestamp', ``query_key``, ``compare_key``, and ``top_count_keys`` are included, if present. (Optional, list of strings, default all fields) +fields +^^^^^^ + +``fields``: A list of fields that should be included in query results and passed to rule types and alerts. If ``_source_enabled`` is False, +only these fields and those from ``include`` are included. When ``_source_enabled`` is True, these are in addition to source. This is used +for runtime fields, script fields, etc. This only works with Elasticsearch version 7.11 and newer. (Optional, list of strings, no default) + top_count_keys ^^^^^^^^^^^^^^ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index a03c2ff6..b702ad22 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -363,6 +363,9 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): query['stored_fields'] = rule['include'] extra_args = {} + if rule.get('fields', None) is not None: + query['fields'] = rule['fields'] + try: if scroll: res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 2366a144..994beddf 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -347,6 +347,7 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('description', "") rule.setdefault('jinja_root_name', "_data") rule.setdefault('query_timezone', "") + rule.setdefault('fields', None) # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() @@ -393,6 +394,9 @@ def _dt_to_ts_with_format(dt): if 'include' in rule and type(rule['include']) != list: raise EAException('include option must be a list') + if 'fields' in rule and rule['fields'] is not None and type(rule['fields']) != list: + raise EAException('fields option must be a list') + raw_query_key = rule.get('query_key') if isinstance(raw_query_key, list): if len(raw_query_key) > 1: diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index dd721bc9..186e8859 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -255,6 +255,7 @@ properties: items: *filter include: {type: array, items: {type: string}} + fields: {type: array, item: {type: string}} top_count_keys: {type: array, items: {type: string}} top_count_number: {type: integer} raw_count_keys: {type: boolean} From af567f90e59f7e666343929fdc1363b82d8aa94b Mon Sep 17 00:00:00 2001 From: Ryan Goggin Date: Wed, 7 Jun 2023 08:08:10 -0400 Subject: [PATCH 07/24] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66074c99..703c4a66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - TBD ## New features -- TBD +- Add `fields` parameter to rules to be able to pull in runtimes fields, and more. [#1193](https://github.com/jertel/elastalert2/pull/1193) - @Goggin ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu From d703544489a0598893ad8c15be20af6e2dbd8207 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 8 Jun 2023 00:59:59 +0900 Subject: [PATCH 08/24] Update pylint, pytest, pytest-cov, pytest-xdist, sphinx pylint 2.17.0 to 2.17.4 pytest 7.2.2 to 7.3.1 pytest-cov 4.0.0 to 4.1.0 pytest-xdist 3.2.0 to 3.3.1 sphinx 6.13 to 6.2.1 sphinx_rtd_theme == 1.2.2 --- requirements-dev.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 70d9ef83..3a643416 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,11 +3,11 @@ flake8 flake8-absolute-import m2r2 pre-commit -pylint==2.17.0 -pytest==7.2.2 -pytest-cov==4.0.0 -pytest-xdist==3.2.0 +pylint==2.17.4 +pytest==7.3.1 +pytest-cov==4.1.0 +pytest-xdist==3.3.1 setuptools -sphinx==6.1.3 -sphinx_rtd_theme +sphinx==6.2.1 +sphinx_rtd_theme==1.2.2 tox==3.28.0 From b4f22cd9b7b23248fc9c5e715758505658b0eca6 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 8 Jun 2023 01:03:59 +0900 Subject: [PATCH 09/24] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be19edcd..b43c6223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - [Kubernetes] Fix mistake introduced in #1141 related to initContainers - [#1145](https://github.com/jertel/elastalert2/pull/1145) - @aturpin1789 - Add support for Kibana 8.7 for Kibana Discover - [#1153](https://github.com/jertel/elastalert2/pull/1153) - @nsano-rururu - [Docs] Add documentation for accessing subfields of array items - [#1166](https://github.com/jertel/elastalert2/pull/1166) - @jertel +- Upgrade pylint 2.17.0 to 2.17.4, pytest 7.2.2 to 7.3.1, pytest-xdist 3.2.0 to 3.3.1, sphinx 6.1.3 to 6.2.1, sphinx_rtd_theme == 1.2.2 - [#1194](https://github.com/jertel/elastalert2/pull/1194) - @nsano-rururu # 2.10.1 From 0d99b2901a099d3e4835565795d173520242a9b9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 7 Jun 2023 12:07:02 -0400 Subject: [PATCH 10/24] Move changelog entry to TBD section --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43c6223..70193b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu +- Upgrade pylint 2.17.0 to 2.17.4, pytest 7.2.2 to 7.3.1, pytest-xdist 3.2.0 to 3.3.1, sphinx 6.1.3 to 6.2.1, sphinx_rtd_theme == 1.2.2 - [#1194](https://github.com/jertel/elastalert2/pull/1194) - @nsano-rururu # 2.11.0 @@ -21,7 +22,6 @@ - [Kubernetes] Fix mistake introduced in #1141 related to initContainers - [#1145](https://github.com/jertel/elastalert2/pull/1145) - @aturpin1789 - Add support for Kibana 8.7 for Kibana Discover - [#1153](https://github.com/jertel/elastalert2/pull/1153) - @nsano-rururu - [Docs] Add documentation for accessing subfields of array items - [#1166](https://github.com/jertel/elastalert2/pull/1166) - @jertel -- Upgrade pylint 2.17.0 to 2.17.4, pytest 7.2.2 to 7.3.1, pytest-xdist 3.2.0 to 3.3.1, sphinx 6.1.3 to 6.2.1, sphinx_rtd_theme == 1.2.2 - [#1194](https://github.com/jertel/elastalert2/pull/1194) - @nsano-rururu # 2.10.1 From f8ae6786c7284bc705084c5dbffc0955d7207076 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 7 Jun 2023 13:25:59 -0400 Subject: [PATCH 11/24] correct unit tests, avoid count query in test-rule tool when eql used --- elastalert/eql.py | 1 + elastalert/test_rule.py | 49 +++++++++++++++++++++++++---------------- tests/eql_test.py | 27 ++++++----------------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/elastalert/eql.py b/elastalert/eql.py index 9897dda2..5503be2a 100644 --- a/elastalert/eql.py +++ b/elastalert/eql.py @@ -47,5 +47,6 @@ def format_results(results): # relabel events as hits, for consistency events = hits.pop('events') hits['hits'] = events + results['eql'] = True return results \ No newline at end of file diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index b738bd50..9bb16560 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -211,25 +211,28 @@ def test_file(self, conf): terms = res['hits']['hits'][0]['_source'] - # Get a count of all docs - count_query = ElastAlerter.get_query( - conf['filter'], - starttime=self.starttime, - endtime=self.endtime, - timestamp_field=ts, - to_ts_func=conf['dt_to_ts'], - sort=False - ) - try: - res = es_client.count(index=index, body=count_query, ignore_unavailable=True) - except Exception as e: - print("Error querying Elasticsearch:", file=sys.stderr) - print(repr(e)[:2048], file=sys.stderr) - if self.args.stop_error: - exit(2) - return None + is_eql = res.get('eql') == True + + if not is_eql: + # Get a count of all docs + count_query = ElastAlerter.get_query( + conf['filter'], + starttime=self.starttime, + endtime=self.endtime, + timestamp_field=ts, + to_ts_func=conf['dt_to_ts'], + sort=False + ) + try: + res = es_client.count(index=index, body=count_query, ignore_unavailable=True) + except Exception as e: + print("Error querying Elasticsearch:", file=sys.stderr) + print(repr(e)[:2048], file=sys.stderr) + if self.args.stop_error: + exit(2) + return None - num_hits = res['count'] + num_hits = res['count'] if self.args.formatted_output: self.formatted_output['hits'] = num_hits @@ -237,9 +240,14 @@ def test_file(self, conf): self.formatted_output['terms'] = list(terms.keys()) self.formatted_output['result'] = terms else: + if is_eql: + count = 'N/A' + else: + count = num_hits + print( "Got %s hits from the last %s day%s" - % (num_hits, self.args.days, "s" if self.args.days > 1 else "") + % (count, self.args.days, "s" if self.args.days > 1 else "") ) print("\nAvailable terms in first hit:") print_terms(terms, '') @@ -277,6 +285,9 @@ def test_file(self, conf): return None num_hits = len(res['hits']['hits']) + if is_eql: + self.formatted_output['hits'] = num_hits + if self.args.save: print("Downloaded %s documents to save" % (num_hits)) return res['hits']['hits'] diff --git a/tests/eql_test.py b/tests/eql_test.py index f54ec6c7..9ba894cf 100644 --- a/tests/eql_test.py +++ b/tests/eql_test.py @@ -14,20 +14,7 @@ def test_format_request_without_eql(): def test_format_request_with_eql(): - body = { - 'query': { - 'bool': { - 'filter': { - 'bool': { - 'must': [ - {'eql': 'test query'}, - {'other': 'other filter'}, - ] - } - } - } - } - } + body = eql_body() expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'test query'} assert eql.format_request(body) == expected_body @@ -41,7 +28,6 @@ def eql_body(): 'must': [ {'eql': 'test query'}, {'other': 'other filter'}, - {'eql': 'newer query'}, ] } } @@ -53,6 +39,7 @@ def eql_body(): def test_format_request_with_excessive_eql(): body = eql_body() + body['query']['bool']['filter']['bool']['must'].append({'eql': 'newer query'}) expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'newer query'} assert eql.format_request(body) == expected_body @@ -60,13 +47,13 @@ def test_format_request_with_excessive_eql(): def test_format_results_without_events(): expected_results = {'hits': {'hits': []}} results = expected_results - eql.format_results(results) == expected_results + assert eql.format_results(results) == expected_results def test_format_results_with_events(): - expected_results = {'hits': {'events': [{'foo': 'bar'}]}} - results = {'hits': {'hits': [{'foo': 'bar'}]}} - eql.format_results(results) == expected_results + expected_results = {'hits': {'hits': [{'foo': 'bar'}]}, 'eql': True} + results = {'hits': {'events': [{'foo': 'bar'}]}} + assert eql.format_results(results) == expected_results def init_client(): @@ -113,7 +100,7 @@ def test_search_with_eql(): expected_params = {'from': True} expected_headers = {} - expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'newer query', 'size': 12} + expected_body = {'filter': {'bool': {'must': [{'other': 'other filter'}]}}, 'query': 'test query', 'size': 12} results = {} es_client.transport = mock.Mock() es_client.transport.perform_request.return_value = results From 0d9cfb8c228349177443b828f02a20adf7ae6ee1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 7 Jun 2023 16:23:53 -0400 Subject: [PATCH 12/24] Add note about counts not supported with EQL in elastalert-test-rule --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c8fb449f..703b80dc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -997,6 +997,8 @@ guaranteed to have the exact same results as with Elasticsearch. For example, an and flatline require a minimum elapsed time before they begin alerting, based on their timeframe. In addition, use_count_query and use_terms_query rely on run_every to determine their resolution. This script uses a fixed 5 minute window, which is the same as the default. + Also, EQL filters do not support counts, so the output relating to counts may show N/A (Not Applicable). + .. _ruletypes: From b71804fcde1bb47d7e4546fc2a8c2fca2e8e3ee9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 7 Jun 2023 16:24:55 -0400 Subject: [PATCH 13/24] Add note about counts not supported with EQL in elastalert-test-rule --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70193b13..0ae9ec41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## New features - Add initial support for EQL - [#1189](https://github.com/jertel/elastalert2/pull/1189) - @jertel +- Add EQL support to elastalert-test-rule utility - [#1195](https://github.com/jertel/elastalert2/pull/1195) - @jertel ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu From 8ac580e7106309a907579f1e4fe36d4bfc7a84e3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 7 Jun 2023 16:26:45 -0400 Subject: [PATCH 14/24] Upgrade to Tox 4 --- CHANGELOG.md | 1 + requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70193b13..8160cbed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu - Upgrade pylint 2.17.0 to 2.17.4, pytest 7.2.2 to 7.3.1, pytest-xdist 3.2.0 to 3.3.1, sphinx 6.1.3 to 6.2.1, sphinx_rtd_theme == 1.2.2 - [#1194](https://github.com/jertel/elastalert2/pull/1194) - @nsano-rururu +- Upgrade to Tox 4 - [#1196](https://github.com/jertel/elastalert2/pull/1196) - @jertel # 2.11.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 3a643416..30282426 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,4 +10,4 @@ pytest-xdist==3.3.1 setuptools sphinx==6.2.1 sphinx_rtd_theme==1.2.2 -tox==3.28.0 +tox==4.6.0 From eedd2432a309902f48d79e75c8edc3032a708eff Mon Sep 17 00:00:00 2001 From: Ryan Goggin Date: Thu, 8 Jun 2023 09:26:19 -0400 Subject: [PATCH 15/24] Add tests --- tests/base_test.py | 14 +++++++++++++- tests/loaders_test.py | 10 ++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index 450b593b..c2d48e16 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -101,7 +101,7 @@ def test_query(ea): size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) -def test_query_with_fields(ea): +def test_query_with_stored_fields(ea): ea.rules[0]['_source_enabled'] = False ea.thread_data.current_es.search.return_value = {'hits': {'total': {'value': 0}, 'hits': []}} ea.run_query(ea.rules[0], START, END) @@ -112,6 +112,18 @@ def test_query_with_fields(ea): size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_with_fields(ea): + ea.rules[0]['fields'] = ['test_runtime_field'] + ea.thread_data.current_es.search.return_value = {'hits': {'total': {'value': 0}, 'hits': []}} + ea.run_query(ea.rules[0], START, END) + ea.thread_data.current_es.search.assert_called_with(body={ + 'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['test_runtime_field']}, + index='idx', ignore_unavailable=True, _source_includes=['@timestamp'], + size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) + + def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 2aeb8354..ac389fb6 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -45,7 +45,8 @@ 'use_count_query': True, 'email': 'test@test.test', 'aggregation': {'hours': 2}, - 'include': ['comparekey', '@timestamp']} + 'include': ['comparekey', '@timestamp'], + 'fields': ['test_runtime_field']} test_args = mock.Mock() test_args.config = 'test_config' @@ -275,6 +276,8 @@ def test_load_rules(): assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) assert isinstance(rules['run_every'], datetime.timedelta) + assert isinstance(rules['rules'][0]['fields'], list) + assert 'test_runtime_field' in rules['rules'][0]['fields'] for included_key in ['comparekey', 'testkey', '@timestamp']: assert included_key in rules['rules'][0]['include'] @@ -363,7 +366,10 @@ def test_load_disabled_rules(): def test_raises_on_missing_config(): - optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') + optional_keys = ( + 'aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', + 'name', 'fields' + ) test_rule_copy = copy.deepcopy(test_rule) for key in list(test_rule_copy.keys()): test_rule_copy = copy.deepcopy(test_rule) From 45097def00a4b77008b504345c731fcd54eb3a56 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 8 Jun 2023 10:13:20 -0400 Subject: [PATCH 16/24] correct chrono order of changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65bbe7cc..86717f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ - TBD ## New features -- Add `fields` parameter to rules to be able to pull in runtimes fields, and more. [#1193](https://github.com/jertel/elastalert2/pull/1193) - @Goggin - Add initial support for EQL - [#1189](https://github.com/jertel/elastalert2/pull/1189) - @jertel +- Add `fields` parameter to rules to be able to pull in runtimes fields, and more. [#1193](https://github.com/jertel/elastalert2/pull/1193) - @Goggin ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu From 4cd6d8fc863534e42089e78360d301f3c207f3aa Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 8 Jun 2023 17:15:44 -0400 Subject: [PATCH 17/24] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daaa8e3d..b55df735 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,8 @@ ## New features - Add initial support for EQL - [#1189](https://github.com/jertel/elastalert2/pull/1189) - @jertel -- Add EQL support to elastalert-test-rule utility - [#1195](https://github.com/jertel/elastalert2/pull/1195) - @jertel - Add `fields` parameter to rules to be able to pull in runtimes fields, and more. [#1193](https://github.com/jertel/elastalert2/pull/1193) - @Goggin +- Add EQL support to elastalert-test-rule utility - [#1195](https://github.com/jertel/elastalert2/pull/1195) - @jertel ## Other changes - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu From 982f592cba96b6e17a0109ab68142effaab9859b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 8 Jun 2023 21:21:44 -0400 Subject: [PATCH 18/24] add log line when exiting due to --end param falling in the past --- CHANGELOG.md | 1 + elastalert/elastalert.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55df735..27cc6aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Add support for Kibana 8.8 for Kibana Discover - [#1184](https://github.com/jertel/elastalert2/pull/1184) - @nsano-rururu - Upgrade pylint 2.17.0 to 2.17.4, pytest 7.2.2 to 7.3.1, pytest-xdist 3.2.0 to 3.3.1, sphinx 6.1.3 to 6.2.1, sphinx_rtd_theme == 1.2.2 - [#1194](https://github.com/jertel/elastalert2/pull/1194) - @nsano-rururu - Upgrade to Tox 4 - [#1196](https://github.com/jertel/elastalert2/pull/1196) - @jertel +- Log message when exiting due to --end param being in the past - [#1199](https://github.com/jertel/elastalert2/pull/1199) - @jertel # 2.11.0 diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b702ad22..070716d4 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1132,7 +1132,9 @@ def start(self): if self.args.end: endtime = ts_to_dt(self.args.end) - if next_run.replace(tzinfo=dateutil.tz.tzutc()) > endtime: + dt = next_run.replace(tzinfo=dateutil.tz.tzutc()) + if dt > endtime: + elastalert_logger.info("End time '%s' falls before the next run time '%s', exiting." % (endtime, dt)) exit(0) if next_run < datetime.datetime.utcnow(): From 73d67fb15dd166e3f0ef5655d41c06c537cd0fe0 Mon Sep 17 00:00:00 2001 From: Phillip Boushy Date: Thu, 8 Jun 2023 22:44:12 -0700 Subject: [PATCH 19/24] Correct --days description, add --start and --end --days info said that --start and --end don't exist, but they do. Correct that line and add --start and --end info to the page. --- docs/source/ruletypes.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 35bae980..cd0ea141 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -987,9 +987,22 @@ and missing or misconfigured fields. ``--count-only``: Only find the number of matching documents and list available fields. ElastAlert 2 will not be run and documents will not be downloaded. -``--days N``: Instead of the default 1 day, query N days. For selecting more specific time ranges, you must run ElastAlert 2 itself and use ``--start`` +``--days N``: Instead of the default 1 day, query N days. For selecting more specific time ranges, use ``--start`` and ``--end``. +``--end `` will force ElastAlert 2 to stop querying after the given +time, instead of the default, querying to the present time. This really only +makes sense when running standalone. The timestamp is formatted as +``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone ``YYYY-MM-DDTHH:MM:SS-XX:00`` +(UTC-XX). + +``--start `` will force ElastAlert 2 to begin querying from the given +time, instead of the default, querying from the present. The timestamp should be +ISO8601, e.g. ``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone +``YYYY-MM-DDTHH:MM:SS-08:00`` (PST). Note that if querying over a large date +range, no alerts will be sent until that rule has finished querying over the +entire time period. To force querying from the current time, use "NOW". + ``--save-json FILE``: Save all documents downloaded to a file as JSON. This is useful if you wish to modify data while testing or do offline testing in conjunction with ``--data FILE``. A maximum of 10,000 documents will be downloaded. From 44606c94fd52e7fe87a3b78077831cd3615cdea9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Jun 2023 07:59:37 -0400 Subject: [PATCH 20/24] fix time stamp --- elastalert/elastalert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 070716d4..ae8c93bd 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1132,9 +1132,9 @@ def start(self): if self.args.end: endtime = ts_to_dt(self.args.end) - dt = next_run.replace(tzinfo=dateutil.tz.tzutc()) - if dt > endtime: - elastalert_logger.info("End time '%s' falls before the next run time '%s', exiting." % (endtime, dt)) + next_run_dt = next_run.replace(tzinfo=dateutil.tz.tzutc()) + if next_run_dt > endtime: + elastalert_logger.info("End time '%s' falls before the next run time '%s', exiting." % (endtime, next_run_dt)) exit(0) if next_run < datetime.datetime.utcnow(): From 2b4bd5c624009ae57906fac1266e287d0dc578db Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 9 Jun 2023 12:14:52 -0400 Subject: [PATCH 21/24] Correct start/end parameters --- docs/source/ruletypes.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index cd0ea141..9001aef6 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -990,18 +990,13 @@ and missing or misconfigured fields. ``--days N``: Instead of the default 1 day, query N days. For selecting more specific time ranges, use ``--start`` and ``--end``. -``--end `` will force ElastAlert 2 to stop querying after the given -time, instead of the default, querying to the present time. This really only -makes sense when running standalone. The timestamp is formatted as +``--start `` The starting date/time of the search filter's time range. The timestamp is formatted as ``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone ``YYYY-MM-DDTHH:MM:SS-XX:00`` -(UTC-XX). - -``--start `` will force ElastAlert 2 to begin querying from the given -time, instead of the default, querying from the present. The timestamp should be -ISO8601, e.g. ``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone -``YYYY-MM-DDTHH:MM:SS-08:00`` (PST). Note that if querying over a large date -range, no alerts will be sent until that rule has finished querying over the -entire time period. To force querying from the current time, use "NOW". +(UTC-XX). If ``timeframe`` is specified, defaults to the ending time - timeframe. Otherwise defaults to ending time - 1 day. + +``--end `` The ending date/time of the search filter's time range. The timestamp is formatted as +``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone ``YYYY-MM-DDTHH:MM:SS-XX:00`` +(UTC-XX). Defaults to the current time. ``--save-json FILE``: Save all documents downloaded to a file as JSON. This is useful if you wish to modify data while testing or do offline testing in conjunction with ``--data FILE``. A maximum of 10,000 documents will be downloaded. From 0927c6b29bc456b5ea1c8714daf059e0c994bfa1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 14 Jun 2023 17:25:35 -0400 Subject: [PATCH 22/24] release 2.12.0 --- CHANGELOG.md | 11 +++++++++++ chart/elastalert2/Chart.yaml | 4 ++-- chart/elastalert2/README.md | 2 +- chart/elastalert2/values.yaml | 2 +- docs/source/running_elastalert.rst | 2 +- setup.py | 2 +- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27cc6aca..e4cfaa09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ ## Breaking changes - TBD +## New features +- TBD + +## Other changes +- TBD + +# 2.12.0 + +## Breaking changes +- None + ## New features - Add initial support for EQL - [#1189](https://github.com/jertel/elastalert2/pull/1189) - @jertel - Add `fields` parameter to rules to be able to pull in runtimes fields, and more. [#1193](https://github.com/jertel/elastalert2/pull/1193) - @Goggin diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml index b5ca0798..9dba1499 100644 --- a/chart/elastalert2/Chart.yaml +++ b/chart/elastalert2/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: Automated rule-based alerting for Elasticsearch name: elastalert2 -version: 2.11.0 -appVersion: 2.11.0 +version: 2.12.0 +appVersion: 2.12.0 home: https://github.com/jertel/elastalert2 sources: - https://github.com/jertel/elastalert2 diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index 57341956..10177ebe 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -49,7 +49,7 @@ The command removes all the Kubernetes components associated with the chart and | Parameter | Description | Default | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------| | `image.repository` | docker image | jertel/elastalert2 | -| `image.tag` | docker image tag | 2.11.0 | +| `image.tag` | docker image tag | 2.12.0 | | `image.pullPolicy` | image pull policy | IfNotPresent | | `image.pullSecret` | image pull secret | "" | | `podAnnotations` | Annotations to be added to pods | {} | diff --git a/chart/elastalert2/values.yaml b/chart/elastalert2/values.yaml index b9df80f1..8fb64484 100644 --- a/chart/elastalert2/values.yaml +++ b/chart/elastalert2/values.yaml @@ -39,7 +39,7 @@ image: # docker image repository: jertel/elastalert2 # docker image tag - tag: 2.11.0 + tag: 2.12.0 pullPolicy: IfNotPresent pullSecret: "" diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 2629ff11..4d74c074 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -80,7 +80,7 @@ elastalert2 container image on `Docker Hub Date: Thu, 15 Jun 2023 07:38:22 -0400 Subject: [PATCH 23/24] clarify docs. resolves #1206 --- CHANGELOG.md | 2 +- docs/source/running_elastalert.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4cfaa09..fceb9a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - TBD ## Other changes -- TBD +- [Docs] Clarify docs to state that alert_time_limit should not be 0 - [#xx](https://github.com/jertel/elastalert2/pull/xx) - @jertel # 2.12.0 diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 4d74c074..7c19fe0e 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -292,7 +292,7 @@ connections ``writeback_index`` is the name of the index in which ElastAlert 2 will store data. We will create this index later. -``alert_time_limit`` is the retry window for failed alerts. +``alert_time_limit`` is the retry window for failed alerts. Must be greater than zero. Save the file as ``config.yaml`` From 6a71c4c9057dbd152f33fc00b05b1edd841090f2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 15 Jun 2023 07:40:12 -0400 Subject: [PATCH 24/24] fix PR num --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fceb9a14..9b357333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - TBD ## Other changes -- [Docs] Clarify docs to state that alert_time_limit should not be 0 - [#xx](https://github.com/jertel/elastalert2/pull/xx) - @jertel +- [Docs] Clarify docs to state that alert_time_limit should not be 0 - [#1208](https://github.com/jertel/elastalert2/pull/1208) - @jertel # 2.12.0