#156 Support multiple directories in FileRulesLoader

CHANGELOG.md

@@ -26,6 +26,7 @@
 - Support a footer in alert text - [#133](https://github.com/jertel/elastalert2/pull/133) - @nsano-rururu
 - Support extra message features for Slack and Mattermost - [#140](https://github.com/jertel/elastalert2/pull/140) - @nsano-rururu
 - Support for environment variable substitutions in yaml config files
+- Support for multiple rules directories and fix `..data` Kubernetes/Openshift recursive directories in FileRulesLoader [#157](https://github.com/jertel/elastalert2/pull/157) - @mrfroggg
 
 ## Other changes
 - Fix issue with testing alerts that contain Jinja templates - [#101](https://github.com/jertel/elastalert2/pull/101) - @jertel

config.yaml.example

@@ -1,4 +1,5 @@
 # This is the folder that contains the rule yaml files
+# This can also be a list of directories
 # Any .yaml file will be loaded as a rule
 rules_folder: example_rules
 

docs/source/elastalert.rst

@@ -152,7 +152,7 @@ The environment variable ``ES_USE_SSL`` will override this field.
 ``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes.
 Defaults to ``FileRulesLoader`` if not set.
 
-``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all
+``rules_folder``: The name of the folder or a list of folders which contains rule configuration files. ElastAlert will load all
 files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload
 or remove rules based on their respective config files. (only required when using ``FileRulesLoader``).
 

elastalert/loaders.py

@@ -513,20 +513,28 @@ def get_names(self, conf, use_rule=None):
         # Passing a filename directly can bypass rules_folder and .yaml checks
         if use_rule and os.path.isfile(use_rule):
             return [use_rule]
-        rule_folder = conf['rules_folder']
+
+        # In case of a bad type, convert string to list:
+        rule_folders = conf['rules_folder'] if isinstance(conf['rules_folder'], list) else [conf['rules_folder']]
         rule_files = []
         if 'scan_subdirectories' in conf and conf['scan_subdirectories']:
-            for root, folders, files in os.walk(rule_folder):
-                for filename in files:
-                    if use_rule and use_rule != filename:
-                        continue
-                    if self.is_yaml(filename):
-                        rule_files.append(os.path.join(root, filename))
+            for ruledir in rule_folders:
+                for root, folders, files in os.walk(ruledir):
+                    # Openshift/k8s configmap fix for ..data and ..2021_05..date directories that loop with os.walk()
+                    folders[:] = [d for d in folders if not d.startswith('..')]
+                    for filename in files:
+                        if use_rule and use_rule != filename:
+                            continue
+                        if self.is_yaml(filename):
+                            rule_files.append(os.path.join(root, filename))
         else:
-            for filename in os.listdir(rule_folder):
-                fullpath = os.path.join(rule_folder, filename)
-                if os.path.isfile(fullpath) and self.is_yaml(filename):
-                    rule_files.append(fullpath)
+            for ruledir in rule_folders:
+                if not os.path.isdir(ruledir):
+                    continue
+                for file in os.scandir(ruledir):
+                    fullpath = os.path.join(ruledir, file.name)
+                    if os.path.isfile(fullpath) and self.is_yaml(file.name):
+                        rule_files.append(fullpath)
         return rule_files
 
     def get_hashes(self, conf, use_rule=None):

tests/loaders_test.py

@@ -169,9 +169,9 @@ def test_load_inline_alert_rule():
 def test_file_rules_loader_get_names_recursive():
     conf = {'scan_subdirectories': True, 'rules_folder': 'root'}
     rules_loader = FileRulesLoader(conf)
-    walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)),
-                  ('root/folder_a', (), ('a.yaml', 'ab.yaml')),
-                  ('root/folder_b', (), ('b.yaml',)))
+    walk_paths = (('root', ['folder_a', 'folder_b'], ('rule.yaml',)),
+                  ('root/folder_a', [], ('a.yaml', 'ab.yaml')),
+                  ('root/folder_b', [], ('b.yaml',)))
     with mock.patch('os.walk') as mock_walk:
         mock_walk.return_value = walk_paths
         paths = rules_loader.get_names(conf)
@@ -186,19 +186,26 @@ def test_file_rules_loader_get_names_recursive():
 
 
 def test_file_rules_loader_get_names():
+
+    class MockDirEntry:
+        # os.DirEntry of os.scandir
+        def __init__(self, name):
+            self.name = name
+
     # Check for no subdirectory
     conf = {'scan_subdirectories': False, 'rules_folder': 'root'}
     rules_loader = FileRulesLoader(conf)
-    files = ['badfile', 'a.yaml', 'b.yaml']
+    files = [MockDirEntry(name='badfile'), MockDirEntry('a.yaml'), MockDirEntry('b.yaml')]
 
-    with mock.patch('os.listdir') as mock_list:
-        with mock.patch('os.path.isfile') as mock_path:
-            mock_path.return_value = True
-            mock_list.return_value = files
-            paths = rules_loader.get_names(conf)
+    with mock.patch('os.path.isdir') as mock_dir:
+        with mock.patch('os.scandir') as mock_list:
+            with mock.patch('os.path.isfile') as mock_path:
+                mock_dir.return_value = conf['rules_folder']
+                mock_path.return_value = True
+                mock_list.return_value = files
+                paths = rules_loader.get_names(conf)
 
     paths = [p.replace(os.path.sep, '/') for p in paths]
-
     assert 'root/a.yaml' in paths
     assert 'root/b.yaml' in paths
     assert len(paths) == 2