From ac07f6c5ea071d3e97dfc430d000075fc464fdf6 Mon Sep 17 00:00:00 2001 From: "Charest, Cedric" Date: Thu, 13 May 2021 16:30:43 -0400 Subject: [PATCH] Fixes #156 Support multiple directories in FileRulesLoader --- config.yaml.example | 1 + docs/source/elastalert.rst | 2 +- elastalert/loaders.py | 30 +++++++++++++++++++----------- tests/loaders_test.py | 27 +++++++++++++++++---------- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index 36fd1b12..6679e249 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,4 +1,5 @@ # This is the folder that contains the rule yaml files +# This can also be a list of directories # Any .yaml file will be loaded as a rule rules_folder: example_rules diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 9663c471..ba260b33 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -152,7 +152,7 @@ The environment variable ``ES_USE_SSL`` will override this field. ``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes. Defaults to ``FileRulesLoader`` if not set. -``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all +``rules_folder``: The name of the folder or a list of folders which contains rule configuration files. ElastAlert will load all files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload or remove rules based on their respective config files. (only required when using ``FileRulesLoader``). diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 6228ca05..bd95e3e7 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -513,20 +513,28 @@ def get_names(self, conf, use_rule=None): # Passing a filename directly can bypass rules_folder and .yaml checks if use_rule and os.path.isfile(use_rule): return [use_rule] - rule_folder = conf['rules_folder'] + + # In case of a bad type, convert string to list: + rule_folders = conf['rules_folder'] if isinstance(conf['rules_folder'], list) else [conf['rules_folder']] rule_files = [] if 'scan_subdirectories' in conf and conf['scan_subdirectories']: - for root, folders, files in os.walk(rule_folder): - for filename in files: - if use_rule and use_rule != filename: - continue - if self.is_yaml(filename): - rule_files.append(os.path.join(root, filename)) + for ruledir in rule_folders: + for root, folders, files in os.walk(ruledir): + # Openshift/k8s configmap fix for ..data and ..2021_05..date directories that loop with os.walk() + folders[:] = [d for d in folders if not d.startswith('..')] + for filename in files: + if use_rule and use_rule != filename: + continue + if self.is_yaml(filename): + rule_files.append(os.path.join(root, filename)) else: - for filename in os.listdir(rule_folder): - fullpath = os.path.join(rule_folder, filename) - if os.path.isfile(fullpath) and self.is_yaml(filename): - rule_files.append(fullpath) + for ruledir in rule_folders: + if not os.path.isdir(ruledir): + continue + for file in os.scandir(ruledir): + fullpath = os.path.join(ruledir, file.name) + if os.path.isfile(fullpath) and self.is_yaml(file.name): + rule_files.append(fullpath) return rule_files def get_hashes(self, conf, use_rule=None): diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 5a5ae000..be29dfa9 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -169,9 +169,9 @@ def test_load_inline_alert_rule(): def test_file_rules_loader_get_names_recursive(): conf = {'scan_subdirectories': True, 'rules_folder': 'root'} rules_loader = FileRulesLoader(conf) - walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), - ('root/folder_a', (), ('a.yaml', 'ab.yaml')), - ('root/folder_b', (), ('b.yaml',))) + walk_paths = (('root', ['folder_a', 'folder_b'], ('rule.yaml',)), + ('root/folder_a', [], ('a.yaml', 'ab.yaml')), + ('root/folder_b', [], ('b.yaml',))) with mock.patch('os.walk') as mock_walk: mock_walk.return_value = walk_paths paths = rules_loader.get_names(conf) @@ -186,19 +186,26 @@ def test_file_rules_loader_get_names_recursive(): def test_file_rules_loader_get_names(): + + class MockDirEntry: + # os.DirEntry of os.scandir + def __init__(self, name): + self.name = name + # Check for no subdirectory conf = {'scan_subdirectories': False, 'rules_folder': 'root'} rules_loader = FileRulesLoader(conf) - files = ['badfile', 'a.yaml', 'b.yaml'] + files = [MockDirEntry(name='badfile'), MockDirEntry('a.yaml'), MockDirEntry('b.yaml')] - with mock.patch('os.listdir') as mock_list: - with mock.patch('os.path.isfile') as mock_path: - mock_path.return_value = True - mock_list.return_value = files - paths = rules_loader.get_names(conf) + with mock.patch('os.path.isdir') as mock_dir: + with mock.patch('os.scandir') as mock_list: + with mock.patch('os.path.isfile') as mock_path: + mock_dir.return_value = conf['rules_folder'] + mock_path.return_value = True + mock_list.return_value = files + paths = rules_loader.get_names(conf) paths = [p.replace(os.path.sep, '/') for p in paths] - assert 'root/a.yaml' in paths assert 'root/b.yaml' in paths assert len(paths) == 2