From a6ae87181f09484cb4807bb452f44748dc7742fc Mon Sep 17 00:00:00 2001 From: James Macdonell Date: Sat, 2 Dec 2023 01:47:51 +0000 Subject: [PATCH 1/4] Support hourly index patterns Otherwise there is missing data from top_count_keys for those indexing with logstash-%Y.%m.%d.%H --- elastalert/util.py | 26 +++++++++++++++++--------- tests/util_test.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 269c98bc..4ca09954 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -237,18 +237,26 @@ def format_index(index, start, end, add_extra=False): # Convert to UTC start -= start.utcoffset() end -= end.utcoffset() - original_start = start + + if add_extra: + start -= datetime.timedelta(days=1) + + if "%H" in index: + dt = datetime.timedelta(hours=1) + end = end.replace(second=0, microsecond=0, minute=0) + else: + dt = datetime.timedelta(days=1) + end = end.replace(second=0, microsecond=0, minute=0, hour=0) + indices = set() - while start.date() <= end.date(): + indices.add(start.strftime(index)) + while start <= end: + start += dt indices.add(start.strftime(index)) - start += datetime.timedelta(days=1) - num = len(indices) + if add_extra: - while len(indices) == num: - original_start -= datetime.timedelta(days=1) - new_index = original_start.strftime(index) - assert new_index != index, "You cannot use a static index with search_extra_index" - indices.add(new_index) + if index in indices: + raise EAException("You cannot use a static index {} with search_extra_index".format(index)) return ','.join(indices) diff --git a/tests/util_test.py b/tests/util_test.py index 6256d732..40d6320f 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -245,6 +245,51 @@ def test_format_index(): assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26'] +def test_format_hourly_index(): + pattern = 'logstash-%Y.%m.%d.%H' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + index_csv = format_index(pattern, date, date2, add_extra=False) + indexes = sorted(index_csv.split(',')) + assert indexes == [ + 'logstash-2023.12.01.22', + 'logstash-2023.12.01.23', + 'logstash-2023.12.02.00' + ] + + +def test_format_hourly_index_with_extra_day(): + pattern = 'logstash-%Y.%m.%d.%H' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + index_csv = format_index(pattern, date, date2, add_extra=True) + indexes = sorted(index_csv.split(',')) + + # with add_extra, first will be one day earlier logstash-2023.11.30.22 + expected = [ + 'logstash-2023.11.30.22', + 'logstash-2023.11.30.23', + ] + extra_24_hours_from_add_extra = ["logstash-2023.12.01.{:02d}".format(hour) for hour in range(24)] + expected.extend(extra_24_hours_from_add_extra) + + # with add_extra, last should still include the index contaning date2 + expected.append('logstash-2023.12.02.00') + + assert indexes == expected + + +def test_format_index_with_static_throws_exception(): + pattern = 'my-static-index-name' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + works_when_add_extra_is_false = format_index(pattern, date, date2, add_extra=False) + assert works_when_add_extra_is_false + with pytest.raises(EAException) as e: + _ = format_index(pattern, date, date2, add_extra=True) + assert e.value.args[0] == "You cannot use a static index {} with search_extra_index".format(pattern) + + def test_should_scrolling_continue(): rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1} rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2} From 7e37678e8756cb48bebae9bbbbad052bb993a2e8 Mon Sep 17 00:00:00 2001 From: James Macdonell Date: Sat, 2 Dec 2023 15:43:18 -0800 Subject: [PATCH 2/4] CHANGELOG.md for #1328 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96cc09ff..10990bb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix percentiles aggregation type in Spike Metric Aggregation rules - [#1323](https://github.com/jertel/elastalert2/pull/1323) - @jertel - [Docs] Extend FAQ / troubleshooting section with information on Elasticsearch RBAC - [#1324](https://github.com/jertel/elastalert2/pull/1324) - @chr-b - Upgrade to Python 3.12 - [#1327](https://github.com/jertel/elastalert2/pull/1327) - @jertel +- Support hourly index patterns - [#1328](https://github.com/jertel/elastalert2/pull/1328) - @jmacdone # 2.15.0 From 59ccfb9dc8b281bd2ccb206bbde79c55393ececc Mon Sep 17 00:00:00 2001 From: jmacdone <98667317+jmacdone@users.noreply.github.com> Date: Sat, 2 Dec 2023 16:29:51 -0800 Subject: [PATCH 3/4] Apply suggestions - correct add_extra meaning add_extra intended to include an extra index, not an extra day of indexes Co-authored-by: Jason Ertel --- elastalert/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 4ca09954..e8a9dc34 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -238,16 +238,14 @@ def format_index(index, start, end, add_extra=False): start -= start.utcoffset() end -= end.utcoffset() - if add_extra: - start -= datetime.timedelta(days=1) - if "%H" in index: dt = datetime.timedelta(hours=1) end = end.replace(second=0, microsecond=0, minute=0) else: dt = datetime.timedelta(days=1) end = end.replace(second=0, microsecond=0, minute=0, hour=0) - + if add_extra: + start -= dt indices = set() indices.add(start.strftime(index)) while start <= end: From 312bd354b7ad90cdbf7c7b1ceba5427d47978b67 Mon Sep 17 00:00:00 2001 From: James Macdonell Date: Sat, 2 Dec 2023 16:41:52 -0800 Subject: [PATCH 4/4] Test add_extra as documented --- tests/util_test.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/util_test.py b/tests/util_test.py index 40d6320f..dbb0e8f1 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -258,23 +258,19 @@ def test_format_hourly_index(): ] -def test_format_hourly_index_with_extra_day(): +def test_format_hourly_index_with_extra_index(): pattern = 'logstash-%Y.%m.%d.%H' date = dt('2023-12-01T22:53:01Z') date2 = dt('2023-12-02T00:10:01Z') index_csv = format_index(pattern, date, date2, add_extra=True) indexes = sorted(index_csv.split(',')) - # with add_extra, first will be one day earlier logstash-2023.11.30.22 expected = [ - 'logstash-2023.11.30.22', - 'logstash-2023.11.30.23', + 'logstash-2023.12.01.21', # added by add_extra=True + 'logstash-2023.12.01.22', + 'logstash-2023.12.01.23', + 'logstash-2023.12.02.00', ] - extra_24_hours_from_add_extra = ["logstash-2023.12.01.{:02d}".format(hour) for hour in range(24)] - expected.extend(extra_24_hours_from_add_extra) - - # with add_extra, last should still include the index contaning date2 - expected.append('logstash-2023.12.02.00') assert indexes == expected