Support hourly index patterns

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Fix percentiles aggregation type in Spike Metric Aggregation rules - [#1323](https://github.com/jertel/elastalert2/pull/1323) - @jertel
 - [Docs] Extend FAQ / troubleshooting section with information on Elasticsearch RBAC - [#1324](https://github.com/jertel/elastalert2/pull/1324) - @chr-b
 - Upgrade to Python 3.12 - [#1327](https://github.com/jertel/elastalert2/pull/1327) - @jertel
+- Support hourly index patterns - [#1328](https://github.com/jertel/elastalert2/pull/1328) - @jmacdone
 
 # 2.15.0
 

elastalert/util.py

@@ -237,18 +237,24 @@ def format_index(index, start, end, add_extra=False):
     # Convert to UTC
     start -= start.utcoffset()
     end -= end.utcoffset()
-    original_start = start
+
+    if "%H" in index:
+        dt = datetime.timedelta(hours=1)
+        end = end.replace(second=0, microsecond=0, minute=0)
+    else:
+        dt = datetime.timedelta(days=1)
+        end = end.replace(second=0, microsecond=0, minute=0, hour=0)
+    if add_extra:
+        start -= dt
     indices = set()
-    while start.date() <= end.date():
+    indices.add(start.strftime(index))
+    while start <= end:
+        start += dt
         indices.add(start.strftime(index))
-        start += datetime.timedelta(days=1)
-    num = len(indices)
+
     if add_extra:
-        while len(indices) == num:
-            original_start -= datetime.timedelta(days=1)
-            new_index = original_start.strftime(index)
-            assert new_index != index, "You cannot use a static index with search_extra_index"
-            indices.add(new_index)
+        if index in indices:
+            raise EAException("You cannot use a static index {} with search_extra_index".format(index))
 
     return ','.join(indices)
 

tests/util_test.py

@@ -245,6 +245,47 @@ def test_format_index():
     assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26']
 
 
+def test_format_hourly_index():
+    pattern = 'logstash-%Y.%m.%d.%H'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    index_csv = format_index(pattern, date, date2, add_extra=False)
+    indexes = sorted(index_csv.split(','))
+    assert indexes == [
+        'logstash-2023.12.01.22',
+        'logstash-2023.12.01.23',
+        'logstash-2023.12.02.00'
+        ]
+
+
+def test_format_hourly_index_with_extra_index():
+    pattern = 'logstash-%Y.%m.%d.%H'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    index_csv = format_index(pattern, date, date2, add_extra=True)
+    indexes = sorted(index_csv.split(','))
+
+    expected = [
+        'logstash-2023.12.01.21',  # added by add_extra=True
+        'logstash-2023.12.01.22',
+        'logstash-2023.12.01.23',
+        'logstash-2023.12.02.00',
+    ]
+
+    assert indexes == expected
+
+
+def test_format_index_with_static_throws_exception():
+    pattern = 'my-static-index-name'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    works_when_add_extra_is_false = format_index(pattern, date, date2, add_extra=False)
+    assert works_when_add_extra_is_false
+    with pytest.raises(EAException) as e:
+        _ = format_index(pattern, date, date2, add_extra=True)
+    assert e.value.args[0] == "You cannot use a static index {} with search_extra_index".format(pattern)
+
+
 def test_should_scrolling_continue():
     rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1}
     rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2}