From 6e5cd23196dc8524bb22ed461b65a0d6f23786d9 Mon Sep 17 00:00:00 2001 From: Fodor Zoltan Date: Thu, 13 May 2021 10:03:23 +0300 Subject: [PATCH] #110: Replace yaml loader with one that supports env value substitutions. --- CHANGELOG.md | 1 + Makefile | 5 ++++- elastalert/alerts.py | 4 ++-- elastalert/config.py | 7 ++++--- elastalert/loaders.py | 4 ++-- elastalert/yaml.py | 8 ++++++++ requirements.txt | 1 - setup.py | 1 - tests/alerts_test.py | 24 ++++++++++++------------ tests/config_test.py | 34 ++++++++++++++++++++++++++++++++++ tests/example.config.yaml | 19 +++++++++++++++++++ tests/loaders_test.py | 28 ++++++++++++++-------------- 12 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 elastalert/yaml.py create mode 100644 tests/config_test.py create mode 100644 tests/example.config.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8691e17e..d14b0df7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Added support for alerting via Amazon Simple Email System (SES) - [#105](https://github.com/jertel/elastalert2/pull/105) - @nsano-rururu - Support a footer in alert text - [#133](https://github.com/jertel/elastalert2/pull/133) - @nsano-rururu - Support extra message features for Slack and Mattermost - [#140](https://github.com/jertel/elastalert2/pull/140) - @nsano-rururu +- Support for environment variable substitutions in yaml config files ## Other changes - Fix issue with testing alerts that contain Jinja templates - [#101](https://github.com/jertel/elastalert2/pull/101) - @jertel diff --git a/Makefile b/Makefile index 608c0bb8..cbe63487 100644 --- a/Makefile +++ b/Makefile @@ -21,10 +21,13 @@ test-elasticsearch: test-docker: docker-compose --project-name elastalert build tox - docker-compose --project-name elastalert run --rm tox + docker-compose --project-name elastalert run --rm tox tox -- $(filter-out $@,$(MAKECMDGOALS)) clean: make -C docs clean find . -name '*.pyc' -delete find . -name '__pycache__' -delete rm -rf virtualenv_run .tox .coverage *.egg-info build + +%: + @: diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f7fd1e37..1b508148 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -26,7 +26,6 @@ from jira.exceptions import JIRAError from requests.auth import HTTPProxyAuth from requests.exceptions import RequestException -from staticconf.loader import yaml_loader from texttable import Texttable from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient @@ -38,6 +37,7 @@ from .util import resolve_string from .util import ts_now from .util import ts_to_dt +from .yaml import read_yaml class DateTimeEncoder(json.JSONEncoder): @@ -312,7 +312,7 @@ def get_account(self, account_file): account_file_path = account_file else: account_file_path = os.path.join(os.path.dirname(self.rule['rule_file']), account_file) - account_conf = yaml_loader(account_file_path) + account_conf = read_yaml(account_file_path) if 'user' not in account_conf or 'password' not in account_conf: raise EAException('Account file must have user and password fields') self.user = account_conf['user'] diff --git a/elastalert/config.py b/elastalert/config.py index 1a8bbee3..28684adc 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -4,13 +4,14 @@ import logging.config from envparse import Env -from staticconf.loader import yaml_loader from . import loaders from .util import EAException from .util import elastalert_logger from .util import get_module +from elastalert.yaml import read_yaml + # Required global (config.yaml) configuration options required_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) @@ -45,10 +46,10 @@ def load_conf(args, defaults=None, overwrites=None): """ filename = args.config if filename: - conf = yaml_loader(filename) + conf = read_yaml(filename) else: try: - conf = yaml_loader('config.yaml') + conf = read_yaml('config.yaml') except FileNotFoundError: raise EAException('No --config or config.yaml found') diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 67e2d9a8..6228ca05 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -11,7 +11,6 @@ from jinja2 import Template from jinja2 import Environment from jinja2 import FileSystemLoader -from staticconf.loader import yaml_loader from . import alerts from . import enhancements @@ -29,6 +28,7 @@ from .util import unix_to_dt from .util import unixms_to_dt from .zabbix import ZabbixAlerter +from .yaml import read_yaml class RulesLoader(object): @@ -538,7 +538,7 @@ def get_hashes(self, conf, use_rule=None): def get_yaml(self, filename): try: - return yaml_loader(filename) + return read_yaml(filename) except yaml.scanner.ScannerError as e: raise EAException('Could not parse file %s: %s' % (filename, e)) diff --git a/elastalert/yaml.py b/elastalert/yaml.py new file mode 100644 index 00000000..35810f10 --- /dev/null +++ b/elastalert/yaml.py @@ -0,0 +1,8 @@ +import os +import yaml + + +def read_yaml(path): + with open(path) as f: + yamlContent = os.path.expandvars(f.read()) + return yaml.load(yamlContent, Loader=yaml.FullLoader) diff --git a/requirements.txt b/requirements.txt index 878d0355..c6ad7706 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ mock>=2.0.0 prison>=0.1.2 prometheus_client>=0.10.1 py-zabbix>=1.1.3 -PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.9.0 PyYAML>=5.1 requests>=2.10.0 diff --git a/setup.py b/setup.py index 5423daba..9f618133 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,6 @@ 'prison>=0.1.2', 'prometheus_client>=0.10.1', 'py-zabbix>=1.1.3', - 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.9.0', 'PyYAML>=5.1', 'requests>=2.10.0', diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 46589522..1d16018e 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -204,7 +204,7 @@ def test_email_with_auth(): 'alert_subject': 'Test alert for {0}', 'alert_subject_args': ['test_term'], 'smtp_auth_file': 'file.txt', 'rule_file': '/tmp/foo.yaml'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: - with mock.patch('elastalert.alerts.yaml_loader') as mock_open: + with mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'someone', 'password': 'hunter2'} mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) @@ -226,7 +226,7 @@ def test_email_with_cert_key(): 'alert_subject': 'Test alert for {0}', 'alert_subject_args': ['test_term'], 'smtp_auth_file': 'file.txt', 'smtp_cert_file': 'dummy/cert.crt', 'smtp_key_file': 'dummy/client.key', 'rule_file': '/tmp/foo.yaml'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: - with mock.patch('elastalert.alerts.yaml_loader') as mock_open: + with mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'someone', 'password': 'hunter2'} mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) @@ -1279,7 +1279,7 @@ def test_jira(): mock_priority = mock.Mock(id='5') with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = [] @@ -1310,7 +1310,7 @@ def test_jira(): # Search called if jira_bump_tickets rule['jira_bump_tickets'] = True with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -1326,7 +1326,7 @@ def test_jira(): # Remove a field if jira_ignore_in_title set rule['jira_ignore_in_title'] = 'test_term' with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -1340,7 +1340,7 @@ def test_jira(): # Issue is still created if search_issues throws an exception with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.side_effect = JIRAError @@ -1359,7 +1359,7 @@ def test_jira(): # Check ticket is bumped if it is updated 4 days ago mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4)) with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -1375,7 +1375,7 @@ def test_jira(): # Check ticket is bumped is not bumped if ticket is updated right now mock_issue.fields.updated = str(ts_now()) with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -1410,7 +1410,7 @@ def test_jira(): {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}} ] with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -1483,7 +1483,7 @@ def test_jira_arbitrary_field_support(): ] with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -1524,7 +1524,7 @@ def test_jira_arbitrary_field_support(): rule['jira_nonexistent_field'] = 'nonexistent field value' with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -1540,7 +1540,7 @@ def test_jira_arbitrary_field_support(): rule['jira_watchers'] = 'invalid_watcher' with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ - mock.patch('elastalert.alerts.yaml_loader') as mock_open: + mock.patch('elastalert.alerts.read_yaml') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields diff --git a/tests/config_test.py b/tests/config_test.py new file mode 100644 index 00000000..1b6a16ee --- /dev/null +++ b/tests/config_test.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +import os +import mock +import datetime + +from elastalert.config import load_conf + + +def test_config_loads(): + os.environ['ELASTIC_PASS'] = 'password_from_env' + dir_path = os.path.dirname(os.path.realpath(__file__)) + + test_args = mock.Mock() + test_args.config = dir_path + '/example.config.yaml' + test_args.rule = None + test_args.debug = False + test_args.es_debug_trace = None + + conf = load_conf(test_args) + + assert conf['rules_folder'] == '/opt/elastalert/rules' + assert conf['run_every'] == datetime.timedelta(seconds=10) + assert conf['buffer_time'] == datetime.timedelta(minutes=15) + + assert conf['es_host'] == 'elasticsearch' + assert conf['es_port'] == 9200 + + assert conf['es_username'] == 'elastic' + assert conf['es_password'] == 'password_from_env' + + assert conf['writeback_index'] == 'elastalert_status' + assert conf['writeback_alias'] == 'elastalert_alerts' + + assert conf['alert_time_limit'] == datetime.timedelta(days=2) diff --git a/tests/example.config.yaml b/tests/example.config.yaml new file mode 100644 index 00000000..44609eb2 --- /dev/null +++ b/tests/example.config.yaml @@ -0,0 +1,19 @@ +rules_folder: /opt/elastalert/rules + +run_every: + seconds: 10 + +buffer_time: + minutes: 15 + +es_host: elasticsearch +es_port: 9200 + +es_username: elastic +es_password: $ELASTIC_PASS + +writeback_index: elastalert_status +writeback_alias: elastalert_alerts + +alert_time_limit: + days: 2 diff --git a/tests/loaders_test.py b/tests/loaders_test.py index bb8d3d87..5a5ae000 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -207,9 +207,9 @@ def test_file_rules_loader_get_names(): def test_load_rules(): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.walk') as mock_ls: @@ -233,9 +233,9 @@ def test_load_default_host_port(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.walk') as mock_ls: @@ -253,9 +253,9 @@ def test_load_ssl_env_false(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.listdir') as mock_ls: @@ -272,9 +272,9 @@ def test_load_ssl_env_true(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.listdir') as mock_ls: @@ -291,9 +291,9 @@ def test_load_url_prefix_env(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.listdir') as mock_ls: @@ -309,9 +309,9 @@ def test_load_disabled_rules(): test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['is_enabled'] = False test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.listdir') as mock_ls: @@ -334,9 +334,9 @@ def test_raises_on_missing_config(): if key in optional_keys: continue - with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + with mock.patch('elastalert.config.read_yaml') as mock_conf_open: mock_conf_open.return_value = test_config_copy - with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + with mock.patch('elastalert.loaders.read_yaml') as mock_rule_open: mock_rule_open.return_value = test_rule_copy with mock.patch('os.walk') as mock_walk: mock_walk.return_value = [('', [], ['testrule.yaml'])]