Support hourly index patterns

Otherwise there is missing data from top_count_keys for those indexing with logstash-%Y.%m.%d.%H

elastalert/util.py

@@ -236,18 +236,26 @@ def format_index(index, start, end, add_extra=False):
     # Convert to UTC
     start -= start.utcoffset()
     end -= end.utcoffset()
-    original_start = start
+
+    if add_extra:
+       start -= datetime.timedelta(days=1)
+
+    if "%H" in index:
+        dt = datetime.timedelta(hours=1)
+        end = end.replace(second=0, microsecond=0, minute=0)
+    else:
+        dt = datetime.timedelta(days=1)
+        end = end.replace(second=0, microsecond=0, minute=0, hour=0)
+
     indices = set()
-    while start.date() <= end.date():
+    indices.add(start.strftime(index))
+    while start <= end:
+        start += dt
         indices.add(start.strftime(index))
-        start += datetime.timedelta(days=1)
-    num = len(indices)
+
     if add_extra:
-        while len(indices) == num:
-            original_start -= datetime.timedelta(days=1)
-            new_index = original_start.strftime(index)
-            assert new_index != index, "You cannot use a static index with search_extra_index"
-            indices.add(new_index)
+        if index in indices:
+            raise EAException("You cannot use a static index {} with search_extra_index".format(index))
 
     return ','.join(indices)
 

tests/util_test.py

@@ -245,6 +245,51 @@ def test_format_index():
     assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26']
 
 
+def test_format_hourly_index():
+    pattern = 'logstash-%Y.%m.%d.%H'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    index_csv = format_index(pattern, date, date2, add_extra=False)
+    indexes = sorted(index_csv.split(','))
+    assert indexes == [
+        'logstash-2023.12.01.22',
+        'logstash-2023.12.01.23',
+        'logstash-2023.12.02.00'
+        ]
+
+
+def test_format_hourly_index_with_extra_day():
+    pattern = 'logstash-%Y.%m.%d.%H'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    index_csv = format_index(pattern, date, date2, add_extra=True)
+    indexes = sorted(index_csv.split(','))
+
+    # with add_extra, first will be one day earlier logstash-2023.11.30.22
+    expected = [
+        'logstash-2023.11.30.22',
+        'logstash-2023.11.30.23',
+    ]
+    extra_24_hours_from_add_extra = ["logstash-2023.12.01.{:02d}".format(hour) for hour in range(24)]
+    expected.extend(extra_24_hours_from_add_extra)
+
+    # with add_extra, last should still include the index contaning date2
+    expected.append('logstash-2023.12.02.00')
+
+    assert indexes == expected
+
+
+def test_format_index_with_static_throws_exception():
+    pattern = 'my-static-index-name'
+    date = dt('2023-12-01T22:53:01Z')
+    date2 = dt('2023-12-02T00:10:01Z')
+    works_when_add_extra_is_false = format_index(pattern, date, date2, add_extra=False)
+    assert works_when_add_extra_is_false
+    with pytest.raises(EAException) as e:
+        _ = format_index(pattern, date, date2, add_extra=True)
+    assert e.value.args[0] == "You cannot use a static index {} with search_extra_index".format(pattern)
+
+
 def test_should_scrolling_continue():
     rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1}
     rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2}