diff --git a/CHANGELOG.md b/CHANGELOG.md index 19375c73..c1040d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix percentiles aggregation type in Spike Metric Aggregation rules - [#1323](https://github.com/jertel/elastalert2/pull/1323) - @jertel - [Docs] Extend FAQ / troubleshooting section with information on Elasticsearch RBAC - [#1324](https://github.com/jertel/elastalert2/pull/1324) - @chr-b - Upgrade to Python 3.12 - [#1327](https://github.com/jertel/elastalert2/pull/1327) - @jertel +- Support hourly index patterns - [#1328](https://github.com/jertel/elastalert2/pull/1328) - @jmacdone - Correction in IRIS and GELF alerter [#1331](https://github.com/jertel/elastalert2/pull/1331) - @malinkinsa # 2.15.0 diff --git a/elastalert/util.py b/elastalert/util.py index 269c98bc..e8a9dc34 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -237,18 +237,24 @@ def format_index(index, start, end, add_extra=False): # Convert to UTC start -= start.utcoffset() end -= end.utcoffset() - original_start = start + + if "%H" in index: + dt = datetime.timedelta(hours=1) + end = end.replace(second=0, microsecond=0, minute=0) + else: + dt = datetime.timedelta(days=1) + end = end.replace(second=0, microsecond=0, minute=0, hour=0) + if add_extra: + start -= dt indices = set() - while start.date() <= end.date(): + indices.add(start.strftime(index)) + while start <= end: + start += dt indices.add(start.strftime(index)) - start += datetime.timedelta(days=1) - num = len(indices) + if add_extra: - while len(indices) == num: - original_start -= datetime.timedelta(days=1) - new_index = original_start.strftime(index) - assert new_index != index, "You cannot use a static index with search_extra_index" - indices.add(new_index) + if index in indices: + raise EAException("You cannot use a static index {} with search_extra_index".format(index)) return ','.join(indices) diff --git a/tests/util_test.py b/tests/util_test.py index 6256d732..dbb0e8f1 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -245,6 +245,47 @@ def test_format_index(): assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26'] +def test_format_hourly_index(): + pattern = 'logstash-%Y.%m.%d.%H' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + index_csv = format_index(pattern, date, date2, add_extra=False) + indexes = sorted(index_csv.split(',')) + assert indexes == [ + 'logstash-2023.12.01.22', + 'logstash-2023.12.01.23', + 'logstash-2023.12.02.00' + ] + + +def test_format_hourly_index_with_extra_index(): + pattern = 'logstash-%Y.%m.%d.%H' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + index_csv = format_index(pattern, date, date2, add_extra=True) + indexes = sorted(index_csv.split(',')) + + expected = [ + 'logstash-2023.12.01.21', # added by add_extra=True + 'logstash-2023.12.01.22', + 'logstash-2023.12.01.23', + 'logstash-2023.12.02.00', + ] + + assert indexes == expected + + +def test_format_index_with_static_throws_exception(): + pattern = 'my-static-index-name' + date = dt('2023-12-01T22:53:01Z') + date2 = dt('2023-12-02T00:10:01Z') + works_when_add_extra_is_false = format_index(pattern, date, date2, add_extra=False) + assert works_when_add_extra_is_false + with pytest.raises(EAException) as e: + _ = format_index(pattern, date, date2, add_extra=True) + assert e.value.args[0] == "You cannot use a static index {} with search_extra_index".format(pattern) + + def test_should_scrolling_continue(): rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1} rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2}