Merge pull request #731 from ferozsalam/es8-compatibility

Enable loading ElastAlert2 against a fresh ES8 instance

CHANGELOG.md

@@ -31,6 +31,7 @@
 - [Docs] Update of RuleType Configuration Cheat Sheet - [#707](https://github.com/jertel/elastalert2/pull/707) - @nsano-rururu
 - Pytest 7.0.0 to 7.0.1 - [#710](https://github.com/jertel/elastalert2/pull/710) - @nsano-rururu
 - Fixing jira_transition_to schema bug. Change property type from boolean to string [#721](https://github.com/jertel/elastalert2/pull/721) - @toxisch
+- Begin Elasticsearch 8 support - ElastAlert 2 now supports setup with fresh ES 8 instances, and works with some alert types [#731](https://github.com/jertel/elastalert2/pull/731) - @ferozsalam
 
 # 2.3.0
 

elastalert/__init__.py

@@ -95,6 +95,13 @@ def is_atleastseven(self):
         """
         return int(self.es_version.split(".")[0]) >= 7
 
+    def is_atleasteight(self):
+        """
+        Returns True when the Elasticsearch server version >= 8
+        """
+        return int(self.es_version.split(".")[0]) >= 8
+
+
     def resolve_writeback_index(self, writeback_index, doc_type):
         """ In ES6, you cannot have multiple _types per index,
         therefore we use self.writeback_index as the prefix for the actual

elastalert/create_index.py

@@ -28,7 +28,13 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
     else:
         esversion = esinfo['number']
 
-    es_index_mappings = read_es_index_mappings() if is_atleastsix(esversion) else read_es_index_mappings(5)
+    es_index_mappings = {}
+    if is_atleasteight(esversion):
+        es_index_mappings = read_es_index_mappings()
+    elif is_atleastsix(esversion):
+        es_index_mappings = read_es_index_mappings(6)
+    else:
+        es_index_mappings = read_es_index_mappings(5)
 
     es_index = IndicesClient(es_client)
     if not recreate:
@@ -61,8 +67,18 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
 
     # To avoid a race condition. TODO: replace this with a real check
     time.sleep(2)
-
-    if is_atleastseven(esversion):
+    if is_atleasteight(esversion):
+        es_client.indices.put_mapping(index=ea_index,
+                                      body=es_index_mappings['elastalert'])
+        es_client.indices.put_mapping(index=ea_index + '_status',
+                                      body=es_index_mappings['elastalert_status'])
+        es_client.indices.put_mapping(index=ea_index + '_silence',
+                                      body=es_index_mappings['silence'])
+        es_client.indices.put_mapping(index=ea_index + '_error',
+                                      body=es_index_mappings['elastalert_error'])
+        es_client.indices.put_mapping(index=ea_index + '_past',
+                                      body=es_index_mappings['past_elastalert'])
+    elif is_atleastseven(esversion):
         # TODO remove doc_type completely when elasicsearch client allows doc_type=None
         # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8
         es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
@@ -118,7 +134,7 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
     print('Done!')
 
 
-def read_es_index_mappings(es_version=6):
+def read_es_index_mappings(es_version=8):
     print('Reading Elastic {0} index mappings:'.format(es_version))
     return {
         'silence': read_es_index_mapping('silence', es_version),
@@ -150,6 +166,8 @@ def is_atleastsixtwo(es_version):
 def is_atleastseven(es_version):
     return int(es_version.split(".")[0]) >= 7
 
+def is_atleasteight(es_version):
+    return int(es_version.split(".")[0]) >= 8
 
 def main():
     parser = argparse.ArgumentParser()

elastalert/es_mappings/8/elastalert.json

@@ -0,0 +1,39 @@
+{
+  "numeric_detection": true,
+  "date_detection": false,
+  "dynamic_templates": [
+    {
+      "strings_as_keyword": {
+        "mapping": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "match_mapping_type": "string"
+      }
+    }
+  ],
+  "properties": {
+    "rule_name": {
+      "type": "keyword"
+    },
+    "@timestamp": {
+      "type": "date",
+      "format": "date_optional_time"
+    },
+    "alert_time": {
+      "type": "date",
+      "format": "date_optional_time"
+    },
+    "match_time": {
+      "type": "date",
+      "format": "date_optional_time"
+    },
+    "match_body": {
+      "enabled": "false",
+      "type": "object"
+    },
+    "aggregate_id": {
+      "type": "keyword"
+    }
+  }
+}

elastalert/es_mappings/8/elastalert_error.json

@@ -0,0 +1,12 @@
+{
+  "properties": {
+    "data": {
+      "type": "object",
+      "enabled": "false"
+    },
+    "@timestamp": {
+      "type": "date",
+      "format": "date_optional_time"
+    }
+  }
+}

elastalert/es_mappings/8/elastalert_status.json

@@ -0,0 +1,11 @@
+{
+  "properties": {
+    "rule_name": {
+      "type": "keyword"
+    },
+    "@timestamp": {
+      "type": "date",
+      "format": "date_optional_time"
+    }
+  }
+}

elastalert/es_mappings/8/past_elastalert.json

@@ -0,0 +1,18 @@
+{
+  "properties": {
+    "rule_name": {
+      "type": "keyword"
+    },
+    "match_body": {
+      "type": "object",
+      "enabled": "false"
+    },
+    "@timestamp": {
+      "type": "date",
+      "format": "date_optional_time"
+    },
+    "aggregate_id": {
+      "type": "keyword"
+    }
+  }
+}

elastalert/es_mappings/8/silence.json

@@ -0,0 +1,15 @@
+{
+  "properties": {
+    "rule_name": {
+      "type": "keyword"
+    },
+    "until": {
+      "type": "date",
+      "format": "date_optional_time"
+    },
+    "@timestamp": {
+      "type": "date",
+      "format": "date_optional_time"
+    }
+  }
+}

tests/create_index_test.py

@@ -53,6 +53,12 @@ def test_read_es_6_index_mappings():
     print((json.dumps(mappings, indent=2)))
 
 
+def test_read_es_8_index_mappings():
+    mappings = elastalert.create_index.read_es_index_mappings(8)
+    assert len(mappings) == len(es_mappings)
+    print((json.dumps(mappings, indent=2)))
+
+
 @pytest.mark.parametrize('es_version, expected', [
     ('5.6.0', False),
     ('6.0.0', True),
@@ -144,3 +150,17 @@ def test_is_atleastsixtwo(es_version, expected):
 def test_is_atleastseven(es_version, expected):
     result = elastalert.create_index.is_atleastseven(es_version)
     assert result == expected
+
+
+@pytest.mark.parametrize('es_version, expected', [
+    ('5.6.0', False),
+    ('6.0.0', False),
+    ('6.1.0', False),
+    ('7.0.0', False),
+    ('7.1.0', False),
+    ('7.17.0', False),
+    ('8.0.0', True)
+])
+def test_is_atleasteight(es_version, expected):
+    result = elastalert.create_index.is_atleasteight(es_version)
+    assert result == expected