From 68f6dadce14dd932dea5e51ca68e33e16a5712cf Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 8 Mar 2018 14:48:45 +0000 Subject: [PATCH 01/74] Naive attempt to allow rules to define their own writeback_index suffix --- elastalert/create_index.py | 3 +++ elastalert/elastalert.py | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index ae859406c..1e594b5f7 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -170,6 +170,9 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) + # @TODO Also force ES v6 type indices + # @TODO Add alert specific index + # @TODO Add alert template for writeback_index suffixes if(elasticversion > 5): es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 597d821f0..3cca3aa36 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1369,13 +1369,14 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES + writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, writeback_index) if res and not agg_id: agg_id = res['_id'] @@ -1399,9 +1400,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): + def writeback(self, doc_type, body, index=None): + writeback_index = self.writeback_index if index is None else index + if self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) # ES 2.0 - 2.3 does not support dots in field names. @@ -1636,7 +1637,8 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + res = self.writeback('elastalert', alert_body, writeback_index) # If new aggregation, save _id if res and not agg_id: From 366c7c30e72fd50c4d48ae9facfd3f779e7f1865 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 9 Mar 2018 14:18:35 +0000 Subject: [PATCH 02/74] Added writeback_index suffix template to create index --- elastalert/create_index.py | 5 ++++- elastalert/elastalert.py | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 1e594b5f7..9e7851062 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -174,11 +174,12 @@ def main(): # @TODO Add alert specific index # @TODO Add alert template for writeback_index suffixes if(elasticversion > 5): - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], + 'aliases': {index: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -186,6 +187,8 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', + 'aliases': {index: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 3cca3aa36..088286090 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1369,14 +1369,14 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body, writeback_index) + res = self.writeback('elastalert', alert_body, writeback_index_suffix) if res and not agg_id: agg_id = res['_id'] @@ -1400,8 +1400,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body, index=None): - writeback_index = self.writeback_index if index is None else index + def writeback(self, doc_type, body, index_suffix=None): + if index_suffix is not None: + writeback_index = self.get_six_index(doc_type) + '_' + index_suffix if self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) @@ -1637,8 +1638,8 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) - res = self.writeback('elastalert', alert_body, writeback_index) + writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + res = self.writeback('elastalert', alert_body, writeback_index_suffix) # If new aggregation, save _id if res and not agg_id: From e075a6188a914bc3195ae3bac156725aaae542f1 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 9 Mar 2018 15:13:10 +0000 Subject: [PATCH 03/74] writeback_index now can have a single dynamic timestamp --- elastalert/elastalert.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 088286090..059b8c08c 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -199,6 +199,19 @@ def get_index(rule, starttime=None, endtime=None): else: return index + @staticmethod + def get_writeback_index(rule): + if 'writeback_index' not in rule: + return None + elif '%' not in rule['writeback_index']: + return rule['writeback_index'] + else: + index = rule['writeback_index'] + format_start = index.find('%') + format_end = index.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(index[format_start:format_end]) + return index[:format_start] + ts + index[format_end:] + def get_six_index(self, doc_type): """ In ES6, you cannot have multiple _types per index, therefore we use self.writeback_index as the prefix for the actual @@ -1369,7 +1382,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + writeback_index_suffix = self.get_writeback_index(rule) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) @@ -1401,9 +1414,10 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No return body def writeback(self, doc_type, body, index_suffix=None): + writeback_index = self.writeback_index if index_suffix is not None: - writeback_index = self.get_six_index(doc_type) + '_' + index_suffix - if self.is_atleastsix(): + writeback_index = writeback_index + '_' + index_suffix + elif self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) # ES 2.0 - 2.3 does not support dots in field names. @@ -1638,8 +1652,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) - res = self.writeback('elastalert', alert_body, writeback_index_suffix) + res = self.writeback('elastalert', alert_body, self.get_writeback_index(rule)) # If new aggregation, save _id if res and not agg_id: From f1b419fc11831588eb1d6bfb580f0afd0d274ee8 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 12 Mar 2018 15:48:23 +0000 Subject: [PATCH 04/74] Added config file requirement of writeback_alias and used that when searching alerts Renamed writeback_index to writeback_suffix for rules --- config.yaml.example | 1 + elastalert/config.py | 3 +- elastalert/create_index.py | 21 +++++---- elastalert/elastalert.py | 96 ++++++++++++++++---------------------- elastalert/test_rule.py | 1 + tests/config_test.py | 3 +- tests/conftest.py | 1 + 7 files changed, 60 insertions(+), 66 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index beec38030..d3ebe69fd 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,6 +57,7 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed diff --git a/elastalert/config.py b/elastalert/config.py index 34dd4bd39..8627875de 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,7 +29,8 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', + 'writeback_alias', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 9e7851062..4257506e4 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -32,6 +32,7 @@ def main(): parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( @@ -74,6 +75,7 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None @@ -100,6 +102,9 @@ def main(): index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None else raw_input('Name of existing index to copy? (Default None) ')) @@ -128,7 +133,7 @@ def main(): print("Elastic Version:" + esversion.split(".")[0]) elasticversion = int(esversion.split(".")[0]) - if(elasticversion > 5): + if elasticversion > 5: mapping = {'type': 'keyword'} else: mapping = {'index': 'not_analyzed', 'type': 'string'} @@ -158,7 +163,7 @@ def main(): print('Index ' + index + ' already exists. Skipping index creation.') return None - if (elasticversion > 5): + if elasticversion > 5: es.indices.create(index) es.indices.create(index+'_status') es.indices.create(index+'_silence') @@ -170,16 +175,15 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - # @TODO Also force ES v6 type indices - # @TODO Add alert specific index - # @TODO Add alert template for writeback_index suffixes - if(elasticversion > 5): + if elasticversion > 5: + es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias) es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], - 'aliases': {index: {}}, 'mappings': es_mapping}) + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -187,8 +191,9 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias, body={'filter': {'term': {'_type': 'elastalert'}}}) es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', - 'aliases': {index: {}}, 'mappings': es_mapping}) + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 059b8c08c..0d720b8f7 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -132,6 +132,7 @@ def __init__(self, args): self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -199,32 +200,27 @@ def get_index(rule, starttime=None, endtime=None): else: return index - @staticmethod - def get_writeback_index(rule): - if 'writeback_index' not in rule: - return None - elif '%' not in rule['writeback_index']: - return rule['writeback_index'] - else: - index = rule['writeback_index'] - format_start = index.find('%') - format_end = index.rfind('%') + 2 - ts = datetime.datetime.utcnow().strftime(index[format_start:format_end]) - return index[:format_start] + ts + index[format_end:] - - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ + def get_writeback_index(self, doc_type, rule=None): writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' + if rule is None or 'writeback_suffix' not in rule: + if self.is_atleastsix(): + if doc_type == 'silence': + writeback_index += '_silence' + elif doc_type == 'past_elastalert': + writeback_index += '_past' + elif doc_type == 'elastalert_status': + writeback_index += '_status' + elif doc_type == 'elastalert_error': + writeback_index += '_error' + else: + suffix = rule['writeback_suffix'] + if '%' in rule['writeback_suffix']: + format_start = suffix.find('%') + format_end = suffix.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) + suffix = suffix[:format_start] + ts + suffix[format_end:] + writeback_index += '_' + suffix + return writeback_index @staticmethod @@ -657,13 +653,9 @@ def get_starttime(self, rule): query.update(sort) try: - if self.is_atleastsix(): - index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + index = self.get_writeback_index('elastalert_status') + res = self.writeback_es.search(index=index, doc_type='elastalert_status', + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1081,7 +1073,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1089,8 +1081,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1382,14 +1374,13 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index_suffix = self.get_writeback_index(rule) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body, writeback_index_suffix) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1413,12 +1404,8 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body, index_suffix=None): - writeback_index = self.writeback_index - if index_suffix is not None: - writeback_index = writeback_index + '_' + index_suffix - elif self.is_atleastsix(): - writeback_index = self.get_six_index(doc_type) + def writeback(self, doc_type, body, rule=None): + writeback_index = self.get_writeback_index(doc_type, rule) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: @@ -1463,7 +1450,7 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1000) @@ -1477,6 +1464,7 @@ def send_pending_alerts(self): pending_alerts = self.find_recent_pending_alerts(self.alert_time_limit) for alert in pending_alerts: _id = alert['_id'] + _index = alert['_index'] alert = alert['_source'] try: rule_name = alert.pop('rule_name') @@ -1519,7 +1507,7 @@ def send_pending_alerts(self): # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. @@ -1551,13 +1539,13 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=match['_index'], doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: @@ -1575,7 +1563,7 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1) @@ -1652,7 +1640,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body, self.get_writeback_index(rule)) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1717,13 +1705,9 @@ def is_silenced(self, rule_name): query.update(sort) try: - if(self.is_atleastsix()): - index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + index = self.get_writeback_index('silence') + res = self.writeback_es.search(index=index, doc_type='silence', + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index e6be6004d..510e006bd 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -304,6 +304,7 @@ def load_conf(self, rules, args): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': datetime.timedelta(hours=24), 'old_query_limit': datetime.timedelta(weeks=1), diff --git a/tests/config_test.py b/tests/config_test.py index 640814013..035a32043 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -21,7 +21,8 @@ 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..cd3671b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,6 +95,7 @@ def ea(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), From 23172df4688909a73f22a9fe10734000af598cab Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 12 Mar 2018 16:24:06 +0000 Subject: [PATCH 05/74] writeback_alias added as required global config value writeback_suffix added to an optional rule value create_index.py added --alias option writeback_alias used when search for alerts more complex get_write_index used to generate appropriate writeback index --- config.yaml.example | 1 + elastalert/config.py | 3 +- elastalert/create_index.py | 17 ++++++-- elastalert/elastalert.py | 80 +++++++++++++++++++------------------- elastalert/test_rule.py | 1 + tests/config_test.py | 3 +- tests/conftest.py | 1 + 7 files changed, 61 insertions(+), 45 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index beec38030..d3ebe69fd 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,6 +57,7 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed diff --git a/elastalert/config.py b/elastalert/config.py index 34dd4bd39..8627875de 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,7 +29,8 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', + 'writeback_alias', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables diff --git a/elastalert/create_index.py b/elastalert/create_index.py index ae859406c..4257506e4 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -32,6 +32,7 @@ def main(): parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( @@ -74,6 +75,7 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None @@ -100,6 +102,9 @@ def main(): index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None else raw_input('Name of existing index to copy? (Default None) ')) @@ -128,7 +133,7 @@ def main(): print("Elastic Version:" + esversion.split(".")[0]) elasticversion = int(esversion.split(".")[0]) - if(elasticversion > 5): + if elasticversion > 5: mapping = {'type': 'keyword'} else: mapping = {'index': 'not_analyzed', 'type': 'string'} @@ -158,7 +163,7 @@ def main(): print('Index ' + index + ' already exists. Skipping index creation.') return None - if (elasticversion > 5): + if elasticversion > 5: es.indices.create(index) es.indices.create(index+'_status') es.indices.create(index+'_silence') @@ -170,12 +175,15 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - if(elasticversion > 5): + if elasticversion > 5: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias) + es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -183,6 +191,9 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias, body={'filter': {'term': {'_type': 'elastalert'}}}) + es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 597d821f0..0d720b8f7 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -132,6 +132,7 @@ def __init__(self, args): self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -199,19 +200,27 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ + def get_writeback_index(self, doc_type, rule=None): writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' + if rule is None or 'writeback_suffix' not in rule: + if self.is_atleastsix(): + if doc_type == 'silence': + writeback_index += '_silence' + elif doc_type == 'past_elastalert': + writeback_index += '_past' + elif doc_type == 'elastalert_status': + writeback_index += '_status' + elif doc_type == 'elastalert_error': + writeback_index += '_error' + else: + suffix = rule['writeback_suffix'] + if '%' in rule['writeback_suffix']: + format_start = suffix.find('%') + format_end = suffix.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) + suffix = suffix[:format_start] + ts + suffix[format_end:] + writeback_index += '_' + suffix + return writeback_index @staticmethod @@ -644,13 +653,9 @@ def get_starttime(self, rule): query.update(sort) try: - if self.is_atleastsix(): - index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + index = self.get_writeback_index('elastalert_status') + res = self.writeback_es.search(index=index, doc_type='elastalert_status', + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1068,7 +1073,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1076,8 +1081,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1375,7 +1380,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1399,10 +1404,8 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): - writeback_index = self.get_six_index(doc_type) + def writeback(self, doc_type, body, rule=None): + writeback_index = self.get_writeback_index(doc_type, rule) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: @@ -1447,7 +1450,7 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1000) @@ -1461,6 +1464,7 @@ def send_pending_alerts(self): pending_alerts = self.find_recent_pending_alerts(self.alert_time_limit) for alert in pending_alerts: _id = alert['_id'] + _index = alert['_index'] alert = alert['_source'] try: rule_name = alert.pop('rule_name') @@ -1503,7 +1507,7 @@ def send_pending_alerts(self): # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. @@ -1535,13 +1539,13 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=match['_index'], doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: @@ -1559,7 +1563,7 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1) @@ -1636,7 +1640,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1701,13 +1705,9 @@ def is_silenced(self, rule_name): query.update(sort) try: - if(self.is_atleastsix()): - index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + index = self.get_writeback_index('silence') + res = self.writeback_es.search(index=index, doc_type='silence', + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index e6be6004d..510e006bd 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -304,6 +304,7 @@ def load_conf(self, rules, args): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': datetime.timedelta(hours=24), 'old_query_limit': datetime.timedelta(weeks=1), diff --git a/tests/config_test.py b/tests/config_test.py index 640814013..035a32043 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -21,7 +21,8 @@ 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..cd3671b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,6 +95,7 @@ def ea(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), From e94adc39921a22a164cfee8ebc1c64496e99dd05 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 26 Mar 2018 11:48:21 +0100 Subject: [PATCH 06/74] Updated tests to match changes to code base --- tests/base_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index fbb61ba9b..be2a299f9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -260,7 +260,7 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 @@ -268,7 +268,7 @@ def test_match_with_module_from_pending(ea): # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() @@ -346,9 +346,9 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -512,9 +512,9 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -1092,7 +1092,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ From 3f115f26681b48882a03afdf985c7204579ccb54 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 26 Mar 2018 11:49:09 +0100 Subject: [PATCH 07/74] Updated tests to match changes to code base --- tests/base_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index fbb61ba9b..be2a299f9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -260,7 +260,7 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 @@ -268,7 +268,7 @@ def test_match_with_module_from_pending(ea): # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() @@ -346,9 +346,9 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -512,9 +512,9 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -1092,7 +1092,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ From 3fbf82f91dee57cb97cb0cbb65e654f831b34ef1 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Tue, 17 Apr 2018 11:44:13 +0100 Subject: [PATCH 08/74] Added template check. Deletes if templates exists and index does not --- elastalert/create_index.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 4257506e4..edef17b37 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -162,6 +162,9 @@ def main(): if es_index.exists(index): print('Index ' + index + ' already exists. Skipping index creation.') return None + elif es_index.exists_template(index): + print('Template ' + index + ' already exists. Deleting in preparation for creating indexes.') + es_index.delete_template(index) if elasticversion > 5: es.indices.create(index) From 000ce74330dd890ca662897d16b439817f49bfad Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 11 Jul 2018 10:48:54 +0100 Subject: [PATCH 09/74] Extracting out rule loading so that rules can be stored in others other than on disk --- elastalert/config.py | 464 ++-------------------------------- elastalert/elastalert.py | 33 ++- elastalert/ruleloaders.py | 505 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 536 insertions(+), 466 deletions(-) create mode 100644 elastalert/ruleloaders.py diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..0e21fd42b 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,36 +1,15 @@ # -*- coding: utf-8 -*- -import copy import datetime -import hashlib -import logging -import os import sys -import alerts -import enhancements -import jsonschema -import ruletypes -import yaml -import yaml.scanner +import ruleloaders from envparse import Env -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt -# schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) -# Required global (config.yaml) and local (rule.yaml) configuration options +# Required global (config.yaml) configuration options required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) -required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -42,55 +21,10 @@ env = Env(ES_USE_SSL=bool) -# import rule dependency -import_rules = {} - -# Used to map the names of rules to their classes -rules_mapping = { - 'frequency': ruletypes.FrequencyRule, - 'any': ruletypes.AnyRule, - 'spike': ruletypes.SpikeRule, - 'blacklist': ruletypes.BlacklistRule, - 'whitelist': ruletypes.WhitelistRule, - 'change': ruletypes.ChangeRule, - 'flatline': ruletypes.FlatlineRule, - 'new_term': ruletypes.NewTermsRule, - 'cardinality': ruletypes.CardinalityRule, - 'metric_aggregation': ruletypes.MetricAggregationRule, - 'percentage_match': ruletypes.PercentageMatchRule, -} - -# Used to map names of alerts to their classes -alerts_mapping = { - 'email': alerts.EmailAlerter, - 'jira': alerts.JiraAlerter, - 'opsgenie': OpsGenieAlerter, - 'stomp': alerts.StompAlerter, - 'debug': alerts.DebugAlerter, - 'command': alerts.CommandAlerter, - 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, - 'ms_teams': alerts.MsTeamsAlerter, - 'slack': alerts.SlackAlerter, - 'pagerduty': alerts.PagerDutyAlerter, - 'exotel': alerts.ExotelAlerter, - 'twilio': alerts.TwilioAlerter, - 'victorops': alerts.VictorOpsAlerter, - 'telegram': alerts.TelegramAlerter, - 'gitter': alerts.GitterAlerter, - 'servicenow': alerts.ServiceNowAlerter, - 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter +# Used to map the names of rule loaders to their classes +loader_mapping = { + 'file': ruleloaders.FileRulesLoader } -# A partial ordering of alert types. Relative order will be preserved in the resulting alerts list -# For example, jira goes before email so the ticket # will be added to the resulting email. -alerts_order = { - 'jira': 0, - 'email': 1 -} - -base_config = {} def get_module(module_name): @@ -106,337 +40,15 @@ def get_module(module_name): return module -def load_configuration(filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :return: The rule configuration, a dictionary. - """ - rule = load_rule_yaml(filename) - load_options(rule, conf, filename, args) - load_modules(rule, args) - return rule - - -def load_rule_yaml(filename): - rule = { - 'rule_file': filename, - } - - import_rules.pop(filename, None) # clear `filename` dependency - while True: - try: - loaded = yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - - # Special case for merging filters - if both files specify a filter merge (AND) them - if 'filter' in rule and 'filter' in loaded: - rule['filter'] = loaded['filter'] + rule['filter'] - - loaded.update(rule) - rule = loaded - if 'import' in rule: - # Find the path of the next file. - if os.path.isabs(rule['import']): - import_filename = rule['import'] - else: - import_filename = os.path.join(os.path.dirname(filename), rule['import']) - # set dependencies - rules = import_rules.get(filename, []) - rules.append(import_filename) - import_rules[filename] = rules - filename = import_filename - del(rule['import']) # or we could go on forever! - else: - break - - return rule - - -def load_options(rule, conf, filename, args=None): - """ Converts time objects, sets defaults, and validates some settings. - - :param rule: A dictionary of parsed YAML from a rule config file. - :param conf: The global configuration dictionary, used for populating defaults. - """ - adjust_deprecated_values(rule) - - try: - rule_schema.validate(rule) - except jsonschema.ValidationError as e: - raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) - - try: - # Set all time based parameters - if 'timeframe' in rule: - rule['timeframe'] = datetime.timedelta(**rule['timeframe']) - if 'realert' in rule: - rule['realert'] = datetime.timedelta(**rule['realert']) - else: - if 'aggregation' in rule: - rule['realert'] = datetime.timedelta(minutes=0) - else: - rule['realert'] = datetime.timedelta(minutes=1) - if 'aggregation' in rule and not rule['aggregation'].get('schedule'): - rule['aggregation'] = datetime.timedelta(**rule['aggregation']) - if 'query_delay' in rule: - rule['query_delay'] = datetime.timedelta(**rule['query_delay']) - if 'buffer_time' in rule: - rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) - if 'bucket_interval' in rule: - rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) - if 'exponential_realert' in rule: - rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) - if 'kibana4_start_timedelta' in rule: - rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) - if 'kibana4_end_timedelta' in rule: - rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) - except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) - - # Set defaults, copy defaults from config.yaml - for key, val in base_config.items(): - rule.setdefault(key, val) - rule.setdefault('name', os.path.splitext(filename)[0]) - rule.setdefault('realert', datetime.timedelta(seconds=0)) - rule.setdefault('aggregation', datetime.timedelta(seconds=0)) - rule.setdefault('query_delay', datetime.timedelta(seconds=0)) - rule.setdefault('timestamp_field', '@timestamp') - rule.setdefault('filter', []) - rule.setdefault('timestamp_type', 'iso') - rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') - rule.setdefault('_source_enabled', True) - rule.setdefault('use_local_time', True) - rule.setdefault('description', "") - - # Set timestamp_type conversion function, used when generating queries and processing hits - rule['timestamp_type'] = rule['timestamp_type'].strip().lower() - if rule['timestamp_type'] == 'iso': - rule['ts_to_dt'] = ts_to_dt - rule['dt_to_ts'] = dt_to_ts - elif rule['timestamp_type'] == 'unix': - rule['ts_to_dt'] = unix_to_dt - rule['dt_to_ts'] = dt_to_unix - elif rule['timestamp_type'] == 'unix_ms': - rule['ts_to_dt'] = unixms_to_dt - rule['dt_to_ts'] = dt_to_unixms - elif rule['timestamp_type'] == 'custom': - def _ts_to_dt_with_format(ts): - return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) - - def _dt_to_ts_with_format(dt): - ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) - if 'timestamp_format_expr' in rule: - # eval expression passing 'ts' and 'dt' - return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) - else: - return ts - - rule['ts_to_dt'] = _ts_to_dt_with_format - rule['dt_to_ts'] = _dt_to_ts_with_format - else: - raise EAException('timestamp_type must be one of iso, unix, or unix_ms') - - # Add support for client ssl certificate auth - if 'verify_certs' in conf: - rule.setdefault('verify_certs', conf.get('verify_certs')) - rule.setdefault('ca_certs', conf.get('ca_certs')) - rule.setdefault('client_cert', conf.get('client_cert')) - rule.setdefault('client_key', conf.get('client_key')) - - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - - # Make sure we have required options - if required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(required_locals - frozenset(rule.keys())))) - - if 'include' in rule and type(rule['include']) != list: - raise EAException('include option must be a list') - - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) - - if isinstance(rule.get('aggregation_key'), list): - rule['compound_aggregation_key'] = rule['aggregation_key'] - rule['aggregation_key'] = ','.join(rule['aggregation_key']) - - if isinstance(rule.get('compare_key'), list): - rule['compound_compare_key'] = rule['compare_key'] - rule['compare_key'] = ','.join(rule['compare_key']) - elif 'compare_key' in rule: - rule['compound_compare_key'] = [rule['compare_key']] - # Add QK, CK and timestamp to include - include = rule.get('include', ['*']) - if 'query_key' in rule: - include.append(rule['query_key']) - if 'compound_query_key' in rule: - include += rule['compound_query_key'] - if 'compound_aggregation_key' in rule: - include += rule['compound_aggregation_key'] - if 'compare_key' in rule: - include.append(rule['compare_key']) - if 'compound_compare_key' in rule: - include += rule['compound_compare_key'] - if 'top_count_keys' in rule: - include += rule['top_count_keys'] - include.append(rule['timestamp_field']) - rule['include'] = list(set(include)) - - # Check that generate_kibana_url is compatible with the filters - if rule.get('generate_kibana_link'): - for es_filter in rule.get('filter'): - if es_filter: - if 'not' in es_filter: - es_filter = es_filter['not'] - if 'query' in es_filter: - es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): - raise EAException('generate_kibana_link is incompatible with filters other than term, query_string and range. ' - 'Consider creating a dashboard and using use_kibana_dashboard instead.') - - # Check that doc_type is provided if use_count/terms_query - if rule.get('use_count_query') or rule.get('use_terms_query'): - if 'doc_type' not in rule: - raise EAException('doc_type must be specified.') - - # Check that query_key is set if use_terms_query - if rule.get('use_terms_query'): - if 'query_key' not in rule: - raise EAException('query_key must be specified with use_terms_query') - - # Warn if use_strf_index is used with %y, %M or %D - # (%y = short year, %M = minutes, %D = full date) - if rule.get('use_strftime_index'): - for token in ['%y', '%M', '%D']: - if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime(rule.get('index')))) - - if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): - raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') - - -def load_modules(rule, args=None): - """ Loads things that could be modules. Enhancements, alerts and rule type. """ - # Set match enhancements - match_enhancements = [] - for enhancement_name in rule.get('match_enhancements', []): - if enhancement_name in dir(enhancements): - enhancement = getattr(enhancements, enhancement_name) - else: - enhancement = get_module(enhancement_name) - if not issubclass(enhancement, enhancements.BaseEnhancement): - raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % (enhancement_name)) - match_enhancements.append(enhancement(rule)) - rule['match_enhancements'] = match_enhancements - - # Convert rule type into RuleType object - if rule['type'] in rules_mapping: - rule['type'] = rules_mapping[rule['type']] - else: - rule['type'] = get_module(rule['type']) - if not issubclass(rule['type'], ruletypes.RuleType): - raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) - - # Make sure we have required alert and type options - reqs = rule['type'].required_options - - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) - # Instantiate rule - try: - rule['type'] = rule['type'](rule, args) - except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] - # Instantiate alerts only if we're not in debug mode - # In debug mode alerts are not actually sent so don't bother instantiating them - if not args or not args.debug: - rule['alert'] = load_alerts(rule, alert_field=rule['alert']) - - -def isyaml(filename): - return filename.endswith('.yaml') or filename.endswith('.yml') - - -def get_file_paths(conf, use_rule=None): - # Passing a filename directly can bypass rules_folder and .yaml checks - if use_rule and os.path.isfile(use_rule): - return [use_rule] - rule_folder = conf['rules_folder'] - rule_files = [] - if conf['scan_subdirectories']: - for root, folders, files in os.walk(rule_folder): - for filename in files: - if use_rule and use_rule != filename: - continue - if isyaml(filename): - rule_files.append(os.path.join(root, filename)) - else: - for filename in os.listdir(rule_folder): - fullpath = os.path.join(rule_folder, filename) - if os.path.isfile(fullpath) and isyaml(filename): - rule_files.append(fullpath) - return rule_files - - -def load_alerts(rule, alert_field): - def normalize_config(alert): - """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. - This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): - return alert, rule - elif isinstance(alert, dict): - name, config = iter(alert.items()).next() - config_copy = copy.copy(rule) - config_copy.update(config) # warning, this (intentionally) mutates the rule dict - return name, config_copy - else: - raise EAException() - - def create_alert(alert, alert_config): - alert_class = alerts_mapping.get(alert) or get_module(alert) - if not issubclass(alert_class, alerts.Alerter): - raise EAException('Alert module %s is not a subclass of Alerter' % (alert)) - missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset(alert_config or []) - if missing_options: - raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) - return alert_class(alert_config) - - try: - if type(alert_field) != list: - alert_field = [alert_field] - - alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): alerts_order.get(a, 1)) - # Convert all alerts into Alerter objects - alert_field = [create_alert(a, b) for a, b in alert_field] - - except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] - - return alert_field - - -def load_rules(args): +def load_conf(args): """ Creates a conf dictionary for ElastAlerter. Loads the global - config file and then each rule found in rules_folder. + config file and then each rule found in rules_folder. - :param args: The parsed arguments to ElastAlert - :return: The global configuration, a dictionary. - """ - names = [] + :param args: The parsed arguments to ElastAlert + :return: The global configuration, a dictionary. + """ filename = args.config conf = yaml_loader(filename) - use_rule = args.rule for env_var, conf_var in env_settings.items(): val = env(env_var, None) @@ -451,6 +63,7 @@ def load_rules(args): conf.setdefault('scroll_keepalive', '30s') conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) + conf.setdefault('rules_loader', 'file') # Convert run_every, buffer_time into a timedelta object try: @@ -465,56 +78,11 @@ def load_rules(args): else: conf['old_query_limit'] = datetime.timedelta(weeks=1) except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) + raise EAException('Invalid time format used: %s' % e) - global base_config - base_config = copy.deepcopy(conf) + # Initialise the rule loader and load each rule configuration + rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) + rules_loader = rules_loader_class(conf) + conf['rules_loader'] = rules_loader - # Load each rule configuration file - rules = [] - rule_files = get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = load_configuration(rule_file, conf, args) - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) - - conf['rules'] = rules return conf - - -def get_rule_hashes(conf, use_rule=None): - rule_files = get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = get_rulefile_hash(rule_file) - return rule_mod_times - - -def get_rulefile_hash(rule_file): - rulefile_hash = '' - if os.path.exists(rule_file): - with open(rule_file) as fh: - rulefile_hash = hashlib.sha1(fh.read()).digest() - for import_rule_file in import_rules.get(rule_file, []): - rulefile_hash += get_rulefile_hash(import_rule_file) - return rulefile_hash - - -def adjust_deprecated_values(rule): - # From rename of simple HTTP alerter - if rule.get('type') == 'simple': - rule['type'] = 'post' - if 'simple_proxy' in rule: - rule['http_post_proxy'] = rule['simple_proxy'] - if 'simple_webhook_url' in rule: - rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning('"simple" alerter has been renamed "post" and comptability may be removed in a future release.') diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index ed389cfd6..abcc027bf 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -17,11 +17,8 @@ import dateutil.tz import kibana -import yaml from alerts import DebugAlerter -from config import get_rule_hashes -from config import load_configuration -from config import load_rules +from config import load_conf from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException @@ -127,10 +124,10 @@ def __init__(self, args): tracer.setLevel(logging.INFO) tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) - self.conf = load_rules(self.args) + self.conf = load_conf(self.args) self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules'] + self.rules = self.conf['rules_loader'].load_all(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] @@ -147,7 +144,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) + self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) @@ -967,9 +964,10 @@ def modify_rule_for_ES5(new_rule): new_rule['filter'] = new_filters def load_rule_changes(self): - ''' Using the modification times of rule config files, syncs the running rules - to match the files in rules_folder by removing, adding or reloading rules. ''' - new_rule_hashes = get_rule_hashes(self.conf, self.args.rule) + """ Using the modification times of rule config files, syncs the running rules + to match the files in rules_folder by removing, adding or reloading rules. """ + rules_loader = self.conf['rules_loader'] + new_rule_hashes = rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes for rule_file, hash_value in self.rule_hashes.iteritems(): @@ -981,7 +979,7 @@ def load_rule_changes(self): if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -991,12 +989,11 @@ def load_rule_changes(self): message = 'Could not load rule %s: %s' % (rule_file, e) self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. - with open(rule_file) as f: - try: - rule_yaml = yaml.load(f) - except yaml.scanner.ScannerError: - self.send_notification_email(exception=e) - continue + try: + rule_yaml = rules_loader.load_yaml(rule_file) + except EAException: + self.send_notification_email(exception=e) + continue self.send_notification_email(exception=e, rule=rule_yaml) continue @@ -1019,7 +1016,7 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: diff --git a/elastalert/ruleloaders.py b/elastalert/ruleloaders.py new file mode 100644 index 000000000..fccab7605 --- /dev/null +++ b/elastalert/ruleloaders.py @@ -0,0 +1,505 @@ +# -*- coding: utf-8 -*- +import copy +import datetime +import hashlib +import logging +import os +import sys + +import alerts +import enhancements +import jsonschema +import ruletypes +import yaml +import yaml.scanner +from opsgenie import OpsGenieAlerter +from staticconf.loader import yaml_loader +from config import get_module +from util import dt_to_ts +from util import dt_to_ts_with_format +from util import dt_to_unix +from util import dt_to_unixms +from util import EAException +from util import ts_to_dt +from util import ts_to_dt_with_format +from util import unix_to_dt +from util import unixms_to_dt + + +class RulesLoader(object): + # import rule dependency + import_rules = {} + + # Required local (rule.yaml) configuration options + required_locals = frozenset(['alert', 'type', 'name', 'index']) + + # Used to map the names of rules to their classes + rules_mapping = { + 'frequency': ruletypes.FrequencyRule, + 'any': ruletypes.AnyRule, + 'spike': ruletypes.SpikeRule, + 'blacklist': ruletypes.BlacklistRule, + 'whitelist': ruletypes.WhitelistRule, + 'change': ruletypes.ChangeRule, + 'flatline': ruletypes.FlatlineRule, + 'new_term': ruletypes.NewTermsRule, + 'cardinality': ruletypes.CardinalityRule, + 'metric_aggregation': ruletypes.MetricAggregationRule, + 'percentage_match': ruletypes.PercentageMatchRule, + } + + # Used to map names of alerts to their classes + alerts_mapping = { + 'email': alerts.EmailAlerter, + 'jira': alerts.JiraAlerter, + 'opsgenie': OpsGenieAlerter, + 'stomp': alerts.StompAlerter, + 'debug': alerts.DebugAlerter, + 'command': alerts.CommandAlerter, + 'sns': alerts.SnsAlerter, + 'hipchat': alerts.HipChatAlerter, + 'stride': alerts.StrideAlerter, + 'ms_teams': alerts.MsTeamsAlerter, + 'slack': alerts.SlackAlerter, + 'pagerduty': alerts.PagerDutyAlerter, + 'exotel': alerts.ExotelAlerter, + 'twilio': alerts.TwilioAlerter, + 'victorops': alerts.VictorOpsAlerter, + 'telegram': alerts.TelegramAlerter, + 'gitter': alerts.GitterAlerter, + 'servicenow': alerts.ServiceNowAlerter, + 'alerta': alerts.AlertaAlerter, + 'post': alerts.HTTPPostAlerter + } + + # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list + # For example, jira goes before email so the ticket # will be added to the resulting email. + alerts_order = { + 'jira': 0, + 'email': 1 + } + + base_config = {} + + def __init__(self, conf): + # schema for rule yaml + self.rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + + self.base_config = copy.deepcopy(conf) + + def load_all(self, conf, args): + """ + Load all the rules and return them. + :param conf: Configuration dict + :param args: Arguments dict + :return: List of rules + """ + raise NotImplementedError() + + def get_hashes(self, conf, use_rule=None): + """ + Get hashes of the rules. + :param conf: Configuration dict + :param use_rule: Limit to only specified rule + :return: Dict of rule name to hash + """ + raise NotImplementedError() + + def get_yaml(self, filename): + """ + Get and parse the yaml of the specified rule. + :param filename: Rule to get the yaml + :return: Rule YAML dict + """ + raise NotImplementedError() + + def get_import_rule(self, rule): + """ + :param rule: Rule dict + :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + """ + raise NotImplementedError() + + def load_yaml(self, filename): + """ + Load the rule including all dependency rules. + :param filename: Rule to load + :return: Loaded rule dict + """ + rule = { + 'rule_file': filename, + } + + self.import_rules.pop(filename, None) # clear `filename` dependency + while True: + loaded = self.get_yaml(filename) + + # Special case for merging filters - if both files specify a filter merge (AND) them + if 'filter' in rule and 'filter' in loaded: + rule['filter'] = loaded['filter'] + rule['filter'] + + loaded.update(rule) + rule = loaded + if 'import' in rule: + # Find the path of the next file. + import_filename = self.get_import_rule(rule) + # set dependencies + rules = self.import_rules.get(filename, []) + rules.append(import_filename) + self.import_rules[filename] = rules + filename = import_filename + del (rule['import']) # or we could go on forever! + else: + break + + return rule + + def load_options(self, rule, conf, filename, args=None): + """ Converts time objects, sets defaults, and validates some settings. + + :param rule: A dictionary of parsed YAML from a rule config file. + :param conf: The global configuration dictionary, used for populating defaults. + :param filename: Name of the rule + :param args: Arguments + """ + self.adjust_deprecated_values(rule) + + try: + self.rule_schema.validate(rule) + except jsonschema.ValidationError as e: + raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) + + try: + # Set all time based parameters + if 'timeframe' in rule: + rule['timeframe'] = datetime.timedelta(**rule['timeframe']) + if 'realert' in rule: + rule['realert'] = datetime.timedelta(**rule['realert']) + else: + if 'aggregation' in rule: + rule['realert'] = datetime.timedelta(minutes=0) + else: + rule['realert'] = datetime.timedelta(minutes=1) + if 'aggregation' in rule and not rule['aggregation'].get('schedule'): + rule['aggregation'] = datetime.timedelta(**rule['aggregation']) + if 'query_delay' in rule: + rule['query_delay'] = datetime.timedelta(**rule['query_delay']) + if 'buffer_time' in rule: + rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'bucket_interval' in rule: + rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) + if 'exponential_realert' in rule: + rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) + if 'kibana4_start_timedelta' in rule: + rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) + if 'kibana4_end_timedelta' in rule: + rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + except (KeyError, TypeError) as e: + raise EAException('Invalid time format used: %s' % e) + + # Set defaults, copy defaults from config.yaml + for key, val in self.base_config.items(): + rule.setdefault(key, val) + rule.setdefault('name', os.path.splitext(filename)[0]) + rule.setdefault('realert', datetime.timedelta(seconds=0)) + rule.setdefault('aggregation', datetime.timedelta(seconds=0)) + rule.setdefault('query_delay', datetime.timedelta(seconds=0)) + rule.setdefault('timestamp_field', '@timestamp') + rule.setdefault('filter', []) + rule.setdefault('timestamp_type', 'iso') + rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') + rule.setdefault('_source_enabled', True) + rule.setdefault('use_local_time', True) + rule.setdefault('description', "") + + # Set timestamp_type conversion function, used when generating queries and processing hits + rule['timestamp_type'] = rule['timestamp_type'].strip().lower() + if rule['timestamp_type'] == 'iso': + rule['ts_to_dt'] = ts_to_dt + rule['dt_to_ts'] = dt_to_ts + elif rule['timestamp_type'] == 'unix': + rule['ts_to_dt'] = unix_to_dt + rule['dt_to_ts'] = dt_to_unix + elif rule['timestamp_type'] == 'unix_ms': + rule['ts_to_dt'] = unixms_to_dt + rule['dt_to_ts'] = dt_to_unixms + elif rule['timestamp_type'] == 'custom': + def _ts_to_dt_with_format(ts): + return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) + + def _dt_to_ts_with_format(dt): + ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) + if 'timestamp_format_expr' in rule: + # eval expression passing 'ts' and 'dt' + return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) + else: + return ts + + rule['ts_to_dt'] = _ts_to_dt_with_format + rule['dt_to_ts'] = _dt_to_ts_with_format + else: + raise EAException('timestamp_type must be one of iso, unix, or unix_ms') + + # Add support for client ssl certificate auth + if 'verify_certs' in conf: + rule.setdefault('verify_certs', conf.get('verify_certs')) + rule.setdefault('ca_certs', conf.get('ca_certs')) + rule.setdefault('client_cert', conf.get('client_cert')) + rule.setdefault('client_key', conf.get('client_key')) + + # Set HipChat options from global config + rule.setdefault('hipchat_msg_color', 'red') + rule.setdefault('hipchat_domain', 'api.hipchat.com') + rule.setdefault('hipchat_notify', True) + rule.setdefault('hipchat_from', '') + rule.setdefault('hipchat_ignore_ssl_errors', False) + + # Make sure we have required options + if self.required_locals - frozenset(rule.keys()): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(rule.keys())))) + + if 'include' in rule and type(rule['include']) != list: + raise EAException('include option must be a list') + + if isinstance(rule.get('query_key'), list): + rule['compound_query_key'] = rule['query_key'] + rule['query_key'] = ','.join(rule['query_key']) + + if isinstance(rule.get('aggregation_key'), list): + rule['compound_aggregation_key'] = rule['aggregation_key'] + rule['aggregation_key'] = ','.join(rule['aggregation_key']) + + if isinstance(rule.get('compare_key'), list): + rule['compound_compare_key'] = rule['compare_key'] + rule['compare_key'] = ','.join(rule['compare_key']) + elif 'compare_key' in rule: + rule['compound_compare_key'] = [rule['compare_key']] + # Add QK, CK and timestamp to include + include = rule.get('include', ['*']) + if 'query_key' in rule: + include.append(rule['query_key']) + if 'compound_query_key' in rule: + include += rule['compound_query_key'] + if 'compound_aggregation_key' in rule: + include += rule['compound_aggregation_key'] + if 'compare_key' in rule: + include.append(rule['compare_key']) + if 'compound_compare_key' in rule: + include += rule['compound_compare_key'] + if 'top_count_keys' in rule: + include += rule['top_count_keys'] + include.append(rule['timestamp_field']) + rule['include'] = list(set(include)) + + # Check that generate_kibana_url is compatible with the filters + if rule.get('generate_kibana_link'): + for es_filter in rule.get('filter'): + if es_filter: + if 'not' in es_filter: + es_filter = es_filter['not'] + if 'query' in es_filter: + es_filter = es_filter['query'] + if es_filter.keys()[0] not in ('term', 'query_string', 'range'): + raise EAException( + 'generate_kibana_link is incompatible with filters other than term, query_string and range.' + 'Consider creating a dashboard and using use_kibana_dashboard instead.') + + # Check that doc_type is provided if use_count/terms_query + if rule.get('use_count_query') or rule.get('use_terms_query'): + if 'doc_type' not in rule: + raise EAException('doc_type must be specified.') + + # Check that query_key is set if use_terms_query + if rule.get('use_terms_query'): + if 'query_key' not in rule: + raise EAException('query_key must be specified with use_terms_query') + + # Warn if use_strf_index is used with %y, %M or %D + # (%y = short year, %M = minutes, %D = full date) + if rule.get('use_strftime_index'): + for token in ['%y', '%M', '%D']: + if token in rule.get('index'): + logging.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) + + if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): + raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') + + def load_modules(self, rule, args=None): + """ Loads things that could be modules. Enhancements, alerts and rule type. """ + # Set match enhancements + match_enhancements = [] + for enhancement_name in rule.get('match_enhancements', []): + if enhancement_name in dir(enhancements): + enhancement = getattr(enhancements, enhancement_name) + else: + enhancement = get_module(enhancement_name) + if not issubclass(enhancement, enhancements.BaseEnhancement): + raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % enhancement_name) + match_enhancements.append(enhancement(rule)) + rule['match_enhancements'] = match_enhancements + + # Convert rule type into RuleType object + if rule['type'] in self.rules_mapping: + rule['type'] = self.rules_mapping[rule['type']] + else: + rule['type'] = get_module(rule['type']) + if not issubclass(rule['type'], ruletypes.RuleType): + raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) + + # Make sure we have required alert and type options + reqs = rule['type'].required_options + + if reqs - frozenset(rule.keys()): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) + # Instantiate rule + try: + rule['type'] = rule['type'](rule, args) + except (KeyError, EAException) as e: + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] + # Instantiate alerts only if we're not in debug mode + # In debug mode alerts are not actually sent so don't bother instantiating them + if not args or not args.debug: + rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param filename: The name of a rule configuration file. + :param conf: The global configuration dictionary, used for populating defaults. + :param args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule + + def load_alerts(self, rule, alert_field): + def normalize_config(alert): + """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. + This function normalizes them both to the latter format. """ + if isinstance(alert, basestring): + return alert, rule + elif isinstance(alert, dict): + name, config = iter(alert.items()).next() + config_copy = copy.copy(rule) + config_copy.update(config) # warning, this (intentionally) mutates the rule dict + return name, config_copy + else: + raise EAException() + + def create_alert(alert, alert_config): + alert_class = self.alerts_mapping.get(alert) or get_module(alert) + if not issubclass(alert_class, alerts.Alerter): + raise EAException('Alert module %s is not a subclass of Alerter' % alert) + missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset( + alert_config or []) + if missing_options: + raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) + return alert_class(alert_config) + + try: + if type(alert_field) != list: + alert_field = [alert_field] + + alert_field = [normalize_config(x) for x in alert_field] + alert_field = sorted(alert_field, key=lambda (a, b): self.alerts_order.get(a, 1)) + # Convert all alerts into Alerter objects + alert_field = [create_alert(a, b) for a, b in alert_field] + + except (KeyError, EAException) as e: + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] + + return alert_field + + @staticmethod + def adjust_deprecated_values(rule): + # From rename of simple HTTP alerter + if rule.get('type') == 'simple': + rule['type'] = 'post' + if 'simple_proxy' in rule: + rule['http_post_proxy'] = rule['simple_proxy'] + if 'simple_webhook_url' in rule: + rule['http_post_url'] = rule['simple_webhook_url'] + logging.warning( + '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') + + +class FileRulesLoader(RulesLoader): + def load_all(self, conf, args): + names = [] + use_rule = args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.__get_file_paths(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_hashes(self, conf, use_rule=None): + rule_files = self.__get_file_paths(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.__get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def __get_file_paths(self, conf, use_rule=None): + # Passing a filename directly can bypass rules_folder and .yaml checks + if use_rule and os.path.isfile(use_rule): + return [use_rule] + rule_folder = conf['rules_folder'] + rule_files = [] + if conf['scan_subdirectories']: + for root, folders, files in os.walk(rule_folder): + for filename in files: + if use_rule and use_rule != filename: + continue + if self.is_yaml(filename): + rule_files.append(os.path.join(root, filename)) + else: + for filename in os.listdir(rule_folder): + fullpath = os.path.join(rule_folder, filename) + if os.path.isfile(fullpath) and self.is_yaml(filename): + rule_files.append(fullpath) + return rule_files + + def __get_rule_file_hash(self, rule_file): + rule_file_hash = '' + if os.path.exists(rule_file): + with open(rule_file) as fh: + rule_file_hash = hashlib.sha1(fh.read()).digest() + for import_rule_file in self.import_rules.get(rule_file, []): + rule_file_hash += self.__get_rule_file_hash(import_rule_file) + return rule_file_hash + + @staticmethod + def is_yaml(filename): + return filename.endswith('.yaml') or filename.endswith('.yml') From cda2dae64f3ccb78b54fa7f526c951779c451482 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 10:54:29 +0100 Subject: [PATCH 10/74] Fixes from testing, refactoring of names, simplification of RulesLoader and updated documentation --- docs/source/elastalert.rst | 5 +- docs/source/index.rst | 1 + docs/source/recipes/adding_loaders.rst | 84 +++++++ elastalert/config.py | 32 ++- elastalert/elastalert.py | 2 +- elastalert/{ruleloaders.py => loaders.py} | 167 +++++++------ elastalert/test_rule.py | 73 ++---- elastalert/util.py | 14 ++ tests/alerts_test.py | 77 ++++-- tests/base_test.py | 20 +- tests/conftest.py | 18 +- tests/{config_test.py => loaders_test.py} | 291 ++++++++++++---------- 12 files changed, 471 insertions(+), 313 deletions(-) create mode 100644 docs/source/recipes/adding_loaders.rst rename elastalert/{ruleloaders.py => loaders.py} (90%) rename tests/{config_test.py => loaders_test.py} (58%) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index fd2daec8c..60c031cbf 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -131,9 +131,12 @@ The environment variable ``ES_USE_SSL`` will override this field. ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``. +``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes. +Defaults to ``FileRulesLoader`` if not set. + ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload -or remove rules based on their respective config files. +or remove rules based on their respective config files. (only required when using ``FileRulesLoader``). ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true`` diff --git a/docs/source/index.rst b/docs/source/index.rst index c3f02f535..4219bf13e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: recipes/adding_alerts recipes/writing_filters recipes/adding_enhancements + recipes/adding_loaders recipes/signing_requests Indices and Tables diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst new file mode 100644 index 000000000..c77520551 --- /dev/null +++ b/docs/source/recipes/adding_loaders.rst @@ -0,0 +1,84 @@ +.. _loaders: + +Rules Loaders +======================== + +RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to +gather rules for a particular source. Your RulesLoader needs to implement three member functions, and +will look something like this: + +.. code-block:: python + + class AwesomeNewRulesLoader(RulesLoader): + def get_names(self, conf, use_rule=None): + ... + def get_hashes(self, conf, use_rule=None): + ... + def get_yaml(self, rule): + ... + +You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a +python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``. + +Example +------- + +As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a +modules folder for the loader in the ElastAlert directory. + +.. code-block:: console + + $ mkdir elastalert_modules + $ cd elastalert_modules + $ touch __init__.py + +Now, in a file named ``mongo_loader.py``, add + +.. code-block:: python + + from pymongo import MongoClient + from elastalert.loaders import RulesLoader + + class MongoRulesLoader(RulesLoader): + def __init__(self, conf): + super(MongoRulesLoader, self).__init__(conf) + self.client = MongoClient(conf['mongo_url']) + self.db = client[conf['mongo_db']] + + def get_names(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + rules = [] + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule.name] = rule.yaml + rules.append(rule.yaml) + + return rules + + + def get_hashes(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + hashes = {} + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule.name] = rule.yaml + hashes[rule.name] = rule.hash + + return hashes + + def get_yaml(self, rule): + if rule in self.cache: + return self.cache[rule] + + self.cache[rule] = self.db.rules.find_one({'name': rule}).yaml + return self.cache[rule] + +Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the +default FileRulesLoader, so in your ``elastalert.conf`` file:: + + rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader" + diff --git a/elastalert/config.py b/elastalert/config.py index 0e21fd42b..518915bf3 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- import datetime -import sys -import ruleloaders +import loaders from envparse import Env from staticconf.loader import yaml_loader from util import EAException +from util import get_module # Required global (config.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -23,28 +23,16 @@ # Used to map the names of rule loaders to their classes loader_mapping = { - 'file': ruleloaders.FileRulesLoader + 'file': loaders.FileRulesLoader, } -def get_module(module_name): - """ Loads a module and returns a specific object. - module_name should 'module.file.object'. - Returns object or raises EAException on error. """ - try: - module_path, module_class = module_name.rsplit('.', 1) - base_module = __import__(module_path, globals(), locals(), [module_class]) - module = getattr(base_module, module_class) - except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] - return module - - -def load_conf(args): +def load_conf(args, overrides=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert + :param overrides: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -55,6 +43,9 @@ def load_conf(args): if val is not None: conf[conf_var] = val + for key, value in (overrides if overrides is not None else []): + conf[key] = value + # Make sure we have all required globals if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) @@ -84,5 +75,10 @@ def load_conf(args): rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) rules_loader = rules_loader_class(conf) conf['rules_loader'] = rules_loader + # Make sure we have all the required globals for the loader + # Make sure we have all required globals + if rules_loader.required_globals - frozenset(conf.keys()): + raise EAException( + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(conf.keys())))) return conf diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index abcc027bf..7a1503020 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -127,7 +127,7 @@ def __init__(self, args): self.conf = load_conf(self.args) self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules_loader'].load_all(self.conf, self.args) + self.rules = self.conf['rules_loader'].load(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] diff --git a/elastalert/ruleloaders.py b/elastalert/loaders.py similarity index 90% rename from elastalert/ruleloaders.py rename to elastalert/loaders.py index fccab7605..c4037e997 100644 --- a/elastalert/ruleloaders.py +++ b/elastalert/loaders.py @@ -14,7 +14,7 @@ import yaml.scanner from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from config import get_module +from util import get_module from util import dt_to_ts from util import dt_to_ts_with_format from util import dt_to_unix @@ -30,6 +30,9 @@ class RulesLoader(object): # import rule dependency import_rules = {} + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset([]) + # Required local (rule.yaml) configuration options required_locals = frozenset(['alert', 'type', 'name', 'index']) @@ -83,48 +86,98 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml - self.rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + self.rule_schema = jsonschema.Draft4Validator( + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) self.base_config = copy.deepcopy(conf) - def load_all(self, conf, args): + def load(self, conf, args=None): """ - Load all the rules and return them. - :param conf: Configuration dict - :param args: Arguments dict + Discover and load all the rules as defined in the conf and args. + :param dict conf: Configuration dict + :param dict args: Arguments dict :return: List of rules + :rtype: list + """ + names = [] + use_rule = None if args is None else args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.get_names(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_names(self, conf, use_rule=None): + """ + Return a list of rule names that can be passed to `get_yaml` to retrieve. + :param dict conf: Configuration dict + :param str use_rule: Limit to only specified rule + :return: A list of rule names + :rtype: list """ raise NotImplementedError() def get_hashes(self, conf, use_rule=None): """ - Get hashes of the rules. - :param conf: Configuration dict - :param use_rule: Limit to only specified rule + Discover and get the hashes of all the rules as defined in the conf. + :param dict conf: Configuration + :param str use_rule: Limit to only specified rule :return: Dict of rule name to hash + :rtype: dict """ raise NotImplementedError() def get_yaml(self, filename): """ Get and parse the yaml of the specified rule. - :param filename: Rule to get the yaml + :param str filename: Rule to get the yaml :return: Rule YAML dict + :rtype: dict """ raise NotImplementedError() def get_import_rule(self, rule): """ - :param rule: Rule dict + Retrieve the name of the rule to import. + :param dict rule: Rule dict :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + :rtype: str """ - raise NotImplementedError() + return rule['import'] + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param str filename: The name of a rule configuration file. + :param dict conf: The global configuration dictionary, used for populating defaults. + :param dict args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule def load_yaml(self, filename): """ Load the rule including all dependency rules. - :param filename: Rule to load + :param str filename: Rule to load :return: Loaded rule dict + :rtype: dict """ rule = { 'rule_file': filename, @@ -364,19 +417,6 @@ def load_modules(self, rule, args=None): if not args or not args.debug: rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) - def load_configuration(self, filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :param args: Arguments - :return: The rule configuration, a dictionary. - """ - rule = self.load_yaml(filename) - self.load_options(rule, conf, filename, args) - self.load_modules(rule, args) - return rule - def load_alerts(self, rule, alert_field): def normalize_config(alert): """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. @@ -429,55 +469,17 @@ def adjust_deprecated_values(rule): class FileRulesLoader(RulesLoader): - def load_all(self, conf, args): - names = [] - use_rule = args.rule - - # Load each rule configuration file - rules = [] - rule_files = self.__get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = self.load_configuration(rule_file, conf, args) - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) - - return rules - - def get_hashes(self, conf, use_rule=None): - rule_files = self.__get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = self.__get_rule_file_hash(rule_file) - return rule_mod_times - - def get_yaml(self, filename): - try: - return yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - def get_import_rule(self, rule): - if os.path.isabs(rule['import']): - return rule['import'] - else: - return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset(['rules_folder']) - def __get_file_paths(self, conf, use_rule=None): + def get_names(self, conf, use_rule=None): # Passing a filename directly can bypass rules_folder and .yaml checks if use_rule and os.path.isfile(use_rule): return [use_rule] rule_folder = conf['rules_folder'] rule_files = [] - if conf['scan_subdirectories']: + if 'scan_subdirectories' in conf and conf['scan_subdirectories']: for root, folders, files in os.walk(rule_folder): for filename in files: if use_rule and use_rule != filename: @@ -491,13 +493,38 @@ def __get_file_paths(self, conf, use_rule=None): rule_files.append(fullpath) return rule_files - def __get_rule_file_hash(self, rule_file): + def get_hashes(self, conf, use_rule=None): + rule_files = self.get_names(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + """ + Allow for relative paths to the import rule. + :param dict rule: + :return: Path the import rule + :rtype: str + """ + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def get_rule_file_hash(self, rule_file): rule_file_hash = '' if os.path.exists(rule_file): with open(rule_file) as fh: rule_file_hash = hashlib.sha1(fh.read()).digest() for import_rule_file in self.import_rules.get(rule_file, []): - rule_file_hash += self.__get_rule_file_hash(import_rule_file) + rule_file_hash += self.get_rule_file_hash(import_rule_file) return rule_file_hash @staticmethod diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 654146aea..631b97d54 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -7,7 +7,6 @@ import datetime import json import logging -import os import random import re import string @@ -15,12 +14,8 @@ import argparse import mock -import yaml -import elastalert.config -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rule_yaml +from elastalert.config import load_conf from elastalert.elastalert import ElastAlerter from elastalert.util import elasticsearch_client from elastalert.util import lookup_es_key @@ -211,7 +206,7 @@ def run_elastalert(self, rule, conf, args): # It is needed to prevent unnecessary initialization of unused alerters load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert - load_modules(rule, load_modules_args) + conf['rules_loader'].load_modules(rule, load_modules_args) conf['rules'] = [rule] # If using mock data, make sure it's sorted and find appropriate time range @@ -281,50 +276,6 @@ def get_id(): if errors and args.stop_error: exit(1) - def load_conf(self, rules, args): - """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), - for initializing rules. Also initializes rules. - - :return: the default rule configuration, a dictionary """ - if args.config is not None: - with open(args.config) as fh: - conf = yaml.load(fh) - else: - if os.path.isfile('config.yaml'): - with open('config.yaml') as fh: - conf = yaml.load(fh) - else: - conf = {} - - # Need to convert these parameters to datetime objects - for key in ['buffer_time', 'run_every', 'alert_time_limit', 'old_query_limit']: - if key in conf: - conf[key] = datetime.timedelta(**conf[key]) - - # Mock configuration. This specifies the base values for attributes, unless supplied otherwise. - conf_default = { - 'rules_folder': 'rules', - 'es_host': 'localhost', - 'es_port': 14900, - 'writeback_index': 'wb', - 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), - 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), - 'scroll_keepalive': '30s' - } - - for key in conf_default: - if key not in conf: - conf[key] = conf_default[key] - elastalert.config.base_config = copy.deepcopy(conf) - load_options(rules, conf, args.file) - print("Successfully loaded %s\n" % (rules['name'])) - - return conf - def run_rule_test(self): """ Uses args to run the various components of MockElastAlerter such as loading the file, saving data, loading data, and running. @@ -357,9 +308,25 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - rule_yaml = load_rule_yaml(args.file) + # rule_yaml = load_rule_yaml(args.file) - conf = self.load_conf(rule_yaml, args) + # conf = self.load_conf(rule_yaml, args) + overrides = { + 'rules_folder': 'rules', + 'es_host': 'localhost', + 'es_port': 14900, + 'writeback_index': 'wb', + 'max_query_size': 10000, + 'alert_time_limit': datetime.timedelta(hours=24), + 'old_query_limit': datetime.timedelta(weeks=1), + 'run_every': datetime.timedelta(minutes=5), + 'disable_rules_on_error': False, + 'buffer_time': datetime.timedelta(minutes=45), + 'scroll_keepalive': '30s' + } + conf = load_conf(args, overrides) + rule_yaml = conf['rules_loader'].get_yaml(args.file) + conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: with open(args.json, 'r') as data_file: diff --git a/elastalert/util.py b/elastalert/util.py index 1751cb6a9..a62475fc5 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -3,6 +3,7 @@ import datetime import logging import os +import sys import dateutil.parser import dateutil.tz @@ -15,6 +16,19 @@ elastalert_logger = logging.getLogger('elastalert') +def get_module(module_name): + """ Loads a module and returns a specific object. + module_name should 'module.file.object'. + Returns object or raises EAException on error. """ + try: + module_path, module_class = module_name.rsplit('.', 1) + base_module = __import__(module_path, globals(), locals(), [module_class]) + module = getattr(base_module, module_class) + except (ImportError, AttributeError, ValueError) as e: + raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] + return module + + def new_get_event_ts(ts_field): """ Constructs a lambda that may be called to extract the timestamp field from a given event. diff --git a/tests/alerts_test.py b/tests/alerts_test.py index b5f32f4a6..73c27f6dc 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -21,7 +21,7 @@ from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter from elastalert.alerts import StrideAlerter -from elastalert.config import load_modules +from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add from elastalert.util import ts_now @@ -866,7 +866,8 @@ def test_ms_teams(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -902,7 +903,8 @@ def test_ms_teams_uses_color_and_fixed_width_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -938,7 +940,8 @@ def test_slack_uses_custom_title(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -979,7 +982,8 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): 'slack_webhook_url': ['http://please.dontgohere.slack'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1021,7 +1025,8 @@ def test_slack_uses_custom_slack_channel(): 'slack_channel_override': '#test-alert', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1064,7 +1069,8 @@ def test_http_alerter_with_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1095,7 +1101,8 @@ def test_http_alerter_with_payload_all_values(): 'http_post_all_values': True, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1126,7 +1133,8 @@ def test_http_alerter_without_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1156,7 +1164,8 @@ def test_pagerduty_alerter(): 'pagerduty_client_name': 'ponies inc.', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1187,7 +1196,8 @@ def test_pagerduty_alerter_custom_incident_key(): 'pagerduty_incident_key': 'custom key', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1219,7 +1229,8 @@ def test_pagerduty_alerter_custom_incident_key_with_args(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1252,7 +1263,8 @@ def test_pagerduty_alerter_custom_alert_subject(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1286,7 +1298,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1322,7 +1335,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args_specifying_trigger(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1447,7 +1461,8 @@ def test_stride_plain_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1492,7 +1507,8 @@ def test_stride_underline_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1537,7 +1553,8 @@ def test_stride_bold_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1582,7 +1599,8 @@ def test_stride_strong_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1627,7 +1645,8 @@ def test_stride_hyperlink(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1672,7 +1691,8 @@ def test_stride_html(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1724,7 +1744,8 @@ def test_hipchat_body_size_limit_text(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1751,7 +1772,8 @@ def test_hipchat_body_size_limit_html(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1791,7 +1813,8 @@ def test_alerta_no_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1844,7 +1867,8 @@ def test_alerta_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1886,7 +1910,8 @@ def test_alerta_new_style(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) diff --git a/tests/base_test.py b/tests/base_test.py index 52d7ae1f3..cc72b17e3 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -925,8 +925,8 @@ def test_rule_changes(ea): 'rules/rule3.yaml': 'XXX', 'rules/rule2.yaml': '!@#$'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] mock_hashes.return_value = new_hashes @@ -946,8 +946,8 @@ def test_rule_changes(ea): # A new rule with a conflicting name wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes @@ -959,8 +959,8 @@ def test_rule_changes(ea): # A new rule with is_enabled=False wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -970,8 +970,8 @@ def test_rule_changes(ea): # An old rule which didn't load gets reloaded new_hashes = copy.copy(new_hashes) new_hashes['rules/rule4.yaml'] = 'qwerty' - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1191,8 +1191,8 @@ def test_uncaught_exceptions(ea): # Changing the file should re-enable it ea.rule_hashes = {'blah.yaml': 'abc'} new_hashes = {'blah.yaml': 'def'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [ea.disabled_rules[0]] mock_hashes.return_value = new_hashes ea.load_rule_changes() diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..157a8e4ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,6 +51,14 @@ def __init__(self, host='es', port=14900): self.indices = mock_es_indices_client() +class mock_rule_loader(object): + def __init__(self, conf): + self.base_config = conf + self.load = mock.Mock() + self.get_hashes = mock.Mock() + self.load_configuration = mock.Mock() + + class mock_ruletype(object): def __init__(self): self.add_data = mock.Mock() @@ -100,11 +108,13 @@ def ea(): 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} + conf['rules_loader'] = mock_rule_loader(conf) elastalert.elastalert.elasticsearch_client = mock_es_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} + ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() diff --git a/tests/config_test.py b/tests/loaders_test.py similarity index 58% rename from tests/config_test.py rename to tests/loaders_test.py index f444f0e25..2d317d455 100644 --- a/tests/config_test.py +++ b/tests/loaders_test.py @@ -8,13 +8,9 @@ import elastalert.alerts import elastalert.ruletypes -from elastalert.config import get_file_paths -from elastalert.config import load_configuration -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rules +from elastalert.config import load_conf +from elastalert.loaders import FileRulesLoader from elastalert.util import EAException -from elastalert.config import import_rules test_config = {'rules_folder': 'test_folder', 'run_every': {'minutes': 10}, @@ -48,15 +44,16 @@ def test_import_rules(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['type'] = 'testing.test.RuleType' - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'load_yaml') as mock_open: mock_open.return_value = test_rule_copy # Test that type is imported with mock.patch('__builtin__.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' assert mock_import.call_args_list[0][0][3] == ['RuleType'] @@ -66,12 +63,13 @@ def test_import_rules(): test_rule_copy['alert'] = 'testing2.test2.Alerter' with mock.patch('__builtin__.__import__') as mock_import: mock_import.return_value = elastalert.alerts - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' assert mock_import.call_args_list[0][0][3] == ['Alerter'] def test_import_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -82,9 +80,9 @@ def test_import_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -94,10 +92,11 @@ def test_import_import(): assert rules['filter'] == import_rule['filter'] # check global import_rule dependency - assert import_rules == {'blah.yaml': ['importme.ymlt']} + assert rules_loader.import_rules == {'blah.yaml': ['importme.ymlt']} def test_import_absolute_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -108,9 +107,9 @@ def test_import_absolute_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('/importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -123,6 +122,7 @@ def test_import_absolute_import(): def test_import_filter(): # Check that if a filter is specified the rules are merged: + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -133,13 +133,14 @@ def test_import_filter(): 'filter': [{'term': {'ratchet': 'clank'}}], } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert rules['filter'] == [{'term': {'ratchet': 'clank'}}, {'term': {'key': 'value'}}] def test_load_inline_alert_rule(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['alert'] = [ { @@ -154,34 +155,75 @@ def test_load_inline_alert_rule(): } ] test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [test_config_copy, test_rule_copy] - load_modules(test_rule_copy) + rules_loader.load_modules(test_rule_copy) assert isinstance(test_rule_copy['alert'][0], elastalert.alerts.EmailAlerter) assert isinstance(test_rule_copy['alert'][1], elastalert.alerts.EmailAlerter) assert 'foo@bar.baz' in test_rule_copy['alert'][0].rule['email'] assert 'baz@foo.bar' in test_rule_copy['alert'][1].rule['email'] +def test_file_rules_loader_get_names_recursive(): + conf = {'scan_subdirectories': True, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), + ('root/folder_a', (), ('a.yaml', 'ab.yaml')), + ('root/folder_b', (), ('b.yaml',))) + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = walk_paths + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/rule.yaml' in paths + assert 'root/folder_a/a.yaml' in paths + assert 'root/folder_a/ab.yaml' in paths + assert 'root/folder_b/b.yaml' in paths + assert len(paths) == 4 + + +def test_file_rules_loader_get_names(): + # Check for no subdirectory + conf = {'scan_subdirectories': False, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + files = ['badfile', 'a.yaml', 'b.yaml'] + + with mock.patch('os.listdir') as mock_list: + with mock.patch('os.path.isfile') as mock_path: + mock_path.return_value = True + mock_list.return_value = files + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/a.yaml' in paths + assert 'root/b.yaml' in paths + assert len(paths) == 2 + + def test_load_rules(): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) - assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) - assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) - assert isinstance(rules['run_every'], datetime.timedelta) - for included_key in ['comparekey', 'testkey', '@timestamp']: - assert included_key in rules['rules'][0]['include'] + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) + assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) + assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) + assert isinstance(rules['run_every'], datetime.timedelta) + for included_key in ['comparekey', 'testkey', '@timestamp']: + assert included_key in rules['rules'][0]['include'] - # Assert include doesn't contain duplicates - assert rules['rules'][0]['include'].count('@timestamp') == 1 - assert rules['rules'][0]['include'].count('comparekey') == 1 + # Assert include doesn't contain duplicates + assert rules['rules'][0]['include'].count('@timestamp') == 1 + assert rules['rules'][0]['include'].count('comparekey') == 1 def test_load_default_host_port(): @@ -189,16 +231,19 @@ def test_load_default_host_port(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - # Assert include doesn't contain duplicates - assert rules['es_port'] == 12345 - assert rules['es_host'] == 'elasticsearch.test' + # Assert include doesn't contain duplicates + assert rules['es_port'] == 12345 + assert rules['es_host'] == 'elasticsearch.test' def test_load_ssl_env_false(): @@ -206,15 +251,18 @@ def test_load_ssl_env_false(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is False + assert rules['use_ssl'] is False def test_load_ssl_env_true(): @@ -222,15 +270,18 @@ def test_load_ssl_env_true(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is True + assert rules['use_ssl'] is True def test_load_url_prefix_env(): @@ -238,36 +289,68 @@ def test_load_url_prefix_env(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['es_url_prefix'] == 'es/' + assert rules['es_url_prefix'] == 'es/' def test_load_disabled_rules(): test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['is_enabled'] = False test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + # The rule is not loaded for it has "is_enabled=False" + assert len(rules['rules']) == 0 + + +def test_raises_on_missing_config(): + optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') + test_rule_copy = copy.deepcopy(test_rule) + for key in test_rule_copy.keys(): + test_rule_copy = copy.deepcopy(test_rule) + test_config_copy = copy.deepcopy(test_config) + test_rule_copy.pop(key) + + # Non required keys + if key in optional_keys: + continue - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - # The rule is not loaded for it has "is_enabled=False" - assert len(rules['rules']) == 0 + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + with pytest.raises(EAException, message='key %s should be required' % key): + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + print(rules) def test_compound_query_key(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('use_count_query') test_rule_copy['query_key'] = ['field1', 'field2'] - load_options(test_rule_copy, test_config, 'filename.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') assert 'field1' in test_rule_copy['include'] assert 'field2' in test_rule_copy['include'] assert test_rule_copy['query_key'] == 'field1,field2' @@ -275,33 +358,14 @@ def test_compound_query_key(): def test_name_inference(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('name') - load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') assert test_rule_copy['name'] == 'msmerc woz ere' -def test_raises_on_missing_config(): - optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') - test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): - test_rule_copy = copy.deepcopy(test_rule) - test_config_copy = copy.deepcopy(test_config) - test_rule_copy.pop(key) - - # Non required keys - if key in optional_keys: - continue - - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - with pytest.raises(EAException, message='key %s should be required' % key): - rule = load_rules(test_args) - print(rule) - - def test_raises_on_bad_generate_kibana_filters(): test_rule['generate_kibana_link'] = True bad_filters = [[{'not': {'terms': {'blah': 'blah'}}}], @@ -318,48 +382,15 @@ def test_raises_on_bad_generate_kibana_filters(): # Test that all the good filters work, but fail with a bad filter added for good in good_filters: + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['filter'] = good - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.return_value = test_rule_copy - load_configuration('blah', test_config) + rules_loader.load_configuration('blah', test_config) for bad in bad_filters: test_rule_copy['filter'] = good + bad with pytest.raises(EAException): - load_configuration('blah', test_config) - - -def test_get_file_paths_recursive(): - conf = {'scan_subdirectories': True, 'rules_folder': 'root'} - walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), - ('root/folder_a', (), ('a.yaml', 'ab.yaml')), - ('root/folder_b', (), ('b.yaml',))) - with mock.patch('os.walk') as mock_walk: - mock_walk.return_value = walk_paths - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/rule.yaml' in paths - assert 'root/folder_a/a.yaml' in paths - assert 'root/folder_a/ab.yaml' in paths - assert 'root/folder_b/b.yaml' in paths - assert len(paths) == 4 - - -def test_get_file_paths(): - # Check for no subdirectory - conf = {'scan_subdirectories': False, 'rules_folder': 'root'} - files = ['badfile', 'a.yaml', 'b.yaml'] - - with mock.patch('os.listdir') as mock_list: - with mock.patch('os.path.isfile') as mock_path: - mock_path.return_value = True - mock_list.return_value = files - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/a.yaml' in paths - assert 'root/b.yaml' in paths - assert len(paths) == 2 + rules_loader.load_configuration('blah', test_config) From 19f3d0dfb89244a7e0689519d9ac1c91f35ed5fd Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 11:25:08 +0100 Subject: [PATCH 11/74] Fixing tests after merging with v0.1.33 --- tests/alerts_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 9a929fd90..a64195b78 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1205,7 +1205,8 @@ def test_pagerduty_alerter_v2(): 'pagerduty_v2_payload_source': 'mysql.host.name', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', From e8d4422f4d346e8bb32e761977f52f2f6c47185c Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 11:41:19 +0100 Subject: [PATCH 12/74] converted writeback_alias to optional conf value with default and applied strftime directly to suffix --- elastalert/config.py | 4 ++-- elastalert/elastalert.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 8fb03f71a..3a8632ff6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,8 +29,7 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', - 'writeback_alias', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables @@ -448,6 +447,7 @@ def load_rules(args): if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) conf.setdefault('scroll_keepalive', '30s') conf.setdefault('disable_rules_on_error', True) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 63d5dbccd..78d7525bc 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,12 +213,7 @@ def get_writeback_index(self, doc_type, rule=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = rule['writeback_suffix'] - if '%' in rule['writeback_suffix']: - format_start = suffix.find('%') - format_end = suffix.rfind('%') + 2 - ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) - suffix = suffix[:format_start] + ts + suffix[format_end:] + suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix']) writeback_index += '_' + suffix return writeback_index From e16fcbf25014eb3c38f5acf28d76b595ecf5b8db Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 13:49:58 +0100 Subject: [PATCH 13/74] fixing bug in get hashes in elastalert __init__ --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 446416b3b..551f79c9a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -144,7 +144,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args) + self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) From 2f59d19b9e91d575afd7f74f6e2ca014ee21f14e Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 14:58:30 +0100 Subject: [PATCH 14/74] fixes to testing a rule, EA now stores rules_loader as property --- elastalert/config.py | 9 +++++---- elastalert/elastalert.py | 14 +++++++------- elastalert/test_rule.py | 32 +++++++++++++++----------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 518915bf3..6d803fc60 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -27,12 +27,12 @@ } -def load_conf(args, overrides=None): +def load_conf(args, defaults=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert - :param overrides: Dictionary of conf values to override + :param defaults: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -43,8 +43,9 @@ def load_conf(args, overrides=None): if val is not None: conf[conf_var] = val - for key, value in (overrides if overrides is not None else []): - conf[key] = value + for key, value in (defaults.iteritems() if defaults is not None else []): + if key not in conf: + conf[key] = value # Make sure we have all required globals if required_globals - frozenset(conf.keys()): diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 551f79c9a..2ae23724a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -125,9 +125,10 @@ def __init__(self, args): tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) self.conf = load_conf(self.args) + self.rules_loader = self.conf['rules_loader'] self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules_loader'].load(self.conf, self.args) + self.rules = self.rules_loader.load(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] @@ -144,7 +145,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args.rule) + self.rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) @@ -998,8 +999,7 @@ def modify_rule_for_ES5(new_rule): def load_rule_changes(self): """ Using the modification times of rule config files, syncs the running rules to match the files in rules_folder by removing, adding or reloading rules. """ - rules_loader = self.conf['rules_loader'] - new_rule_hashes = rules_loader.get_hashes(self.conf, self.args.rule) + new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes for rule_file, hash_value in self.rule_hashes.iteritems(): @@ -1011,7 +1011,7 @@ def load_rule_changes(self): if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = rules_loader.load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -1022,7 +1022,7 @@ def load_rule_changes(self): self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. try: - rule_yaml = rules_loader.load_yaml(rule_file) + rule_yaml = self.rules_loader.load_yaml(rule_file) except EAException: self.send_notification_email(exception=e) continue @@ -1048,7 +1048,7 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = rules_loader.load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 79b7f6038..ee936839f 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -207,7 +207,6 @@ def run_elastalert(self, rule, conf, args): load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert conf['rules_loader'].load_modules(rule, load_modules_args) - conf['rules'] = [rule] # If using mock data, make sure it's sorted and find appropriate time range timestamp_field = rule.get('timestamp_field', '@timestamp') @@ -264,13 +263,15 @@ def get_id(): conf['run_every'] = endtime - starttime # Instantiate ElastAlert to use mock config and special rule - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - if args.alert: - client = ElastAlerter(['--verbose']) - else: - client = ElastAlerter(['--debug']) + with mock.patch.object(conf['rules_loader'], 'get_hashes'): + with mock.patch.object(conf['rules_loader'], 'load') as load_rules: + load_rules.return_value = [rule] + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + if args.alert: + client = ElastAlerter(['--verbose']) + else: + client = ElastAlerter(['--debug']) # Replace get_hits_* functions to use mock data if args.json: @@ -341,23 +342,20 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - # rule_yaml = load_rule_yaml(args.file) - - # conf = self.load_conf(rule_yaml, args) - overrides = { + defaults = { 'rules_folder': 'rules', 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), + 'alert_time_limit': {'hours': 24}, + 'old_query_limit': {'weeks': 1}, + 'run_every': {'minutes': 5}, 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), + 'buffer_time': {'minutes': 45}, 'scroll_keepalive': '30s' } - conf = load_conf(args, overrides) + conf = load_conf(args, defaults) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From 515b15a10fc2eb8a7df26d889d044225056dd869 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 18 Jul 2018 10:31:38 +0100 Subject: [PATCH 15/74] Added ability for test rule to override loader to use file loader --- docs/source/recipes/adding_loaders.rst | 15 ++++++++------- elastalert/config.py | 8 ++++++-- elastalert/test_rule.py | 5 ++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst index c77520551..672d42390 100644 --- a/docs/source/recipes/adding_loaders.rst +++ b/docs/source/recipes/adding_loaders.rst @@ -38,12 +38,14 @@ Now, in a file named ``mongo_loader.py``, add from pymongo import MongoClient from elastalert.loaders import RulesLoader + import yaml class MongoRulesLoader(RulesLoader): def __init__(self, conf): super(MongoRulesLoader, self).__init__(conf) self.client = MongoClient(conf['mongo_url']) - self.db = client[conf['mongo_db']] + self.db = self.client[conf['mongo_db']] + self.cache = {} def get_names(self, conf, use_rule=None): if use_rule: @@ -52,12 +54,11 @@ Now, in a file named ``mongo_loader.py``, add rules = [] self.cache = {} for rule in self.db.rules.find(): - self.cache[rule.name] = rule.yaml - rules.append(rule.yaml) + self.cache[rule['name']] = yaml.load(rule['yaml']) + rules.append(rule['name']) return rules - def get_hashes(self, conf, use_rule=None): if use_rule: return [use_rule] @@ -65,8 +66,8 @@ Now, in a file named ``mongo_loader.py``, add hashes = {} self.cache = {} for rule in self.db.rules.find(): - self.cache[rule.name] = rule.yaml - hashes[rule.name] = rule.hash + self.cache[rule['name']] = rule['yaml'] + hashes[rule['name']] = rule['hash'] return hashes @@ -74,7 +75,7 @@ Now, in a file named ``mongo_loader.py``, add if rule in self.cache: return self.cache[rule] - self.cache[rule] = self.db.rules.find_one({'name': rule}).yaml + self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml']) return self.cache[rule] Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the diff --git a/elastalert/config.py b/elastalert/config.py index 6d803fc60..7587eafef 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -27,12 +27,13 @@ } -def load_conf(args, defaults=None): +def load_conf(args, defaults=None, overwrites=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert - :param defaults: Dictionary of conf values to override + :param defaults: Dictionary of default conf values + :param overwrites: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -47,6 +48,9 @@ def load_conf(args, defaults=None): if key not in conf: conf[key] = value + for key, value in (overwrites.iteritems() if overwrites is not None else []): + conf[key] = value + # Make sure we have all required globals if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index ee936839f..f93eee4f6 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -355,7 +355,10 @@ def run_rule_test(self): 'buffer_time': {'minutes': 45}, 'scroll_keepalive': '30s' } - conf = load_conf(args, defaults) + overwrites = { + 'rules_loader': 'file', + } + conf = load_conf(args, defaults, overwrites) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From 420791841773a6608f982974ad9156690c1c1a8f Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 4 Sep 2018 14:46:44 +0200 Subject: [PATCH 16/74] slack_ca_certs from ca_certs --- elastalert/alerts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 8eac5ece7..acbef1d42 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1114,6 +1114,7 @@ def __init__(self, rule): self.slack_text_string = self.rule.get('slack_text_string', '') self.slack_alert_fields = self.rule.get('slack_alert_fields', '') self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) + self.slack_ca_certs = self.rule.get('slack_ca_certs') def format_body(self, body): # https://api.slack.com/docs/formatting @@ -1173,12 +1174,16 @@ def alert(self, matches): for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: + if self.slack_ca_certs: + verify = self.slack_ca_certs + else: + verify = self.slack_ignore_ssl_errors if self.slack_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() payload['channel'] = channel_override response = requests.post( url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.slack_ignore_ssl_errors, + headers=headers, verify=verify, proxies=proxies) warnings.resetwarnings() response.raise_for_status() From 6d71629f6148c5e11b9de16248bb1d4b7a4f0162 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 4 Sep 2018 14:58:46 +0200 Subject: [PATCH 17/74] updated schema.yaml --- elastalert/schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index c3d4509e9..bee18c1d9 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -238,6 +238,7 @@ properties: slack_parse_override: {enum: [none, full]} slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} + slack_ca_certs: {type: string} ### PagerDuty pagerduty_service_key: {type: string} From df481d045c7c0efb9a46c268556ac6300564c19d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 14:29:46 -0800 Subject: [PATCH 18/74] Fixed and merge mater with threading and scheduling --- elastalert/config.py | 3 + elastalert/create_index.py | 2 + elastalert/elastalert.py | 229 ++++++++++++++++++++++++------------- elastalert/enhancements.py | 6 + elastalert/util.py | 13 ++- requirements-dev.txt | 1 + requirements.txt | 7 +- setup.py | 1 + tests/base_test.py | 65 +++++------ tests/conftest.py | 15 ++- 10 files changed, 220 insertions(+), 122 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index d1c27f44a..8a9f73995 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -100,6 +100,7 @@ def get_module(module_name): """ Loads a module and returns a specific object. module_name should 'module.file.object'. Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) try: module_path, module_class = module_name.rsplit('.', 1) base_module = __import__(module_path, globals(), locals(), [module_class]) @@ -195,6 +196,8 @@ def load_options(rule, conf, filename, args=None): rule['query_delay'] = datetime.timedelta(**rule['query_delay']) if 'buffer_time' in rule: rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) if 'bucket_interval' in rule: rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) if 'exponential_realert' in rule: diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b12ee7e5e..953c3a1ee 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -57,6 +57,8 @@ def main(): filename = 'config.yaml' elif os.path.isfile(args.config): filename = args.config + elif os.path.isfile('../config.yaml'): + filename = '../config.yaml' else: filename = '' diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 29cc87568..e889161cd 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -5,8 +5,10 @@ import json import logging import os +import random import signal import sys +import threading import time import timeit import traceback @@ -17,8 +19,10 @@ import dateutil.tz import kibana +import pytz import yaml from alerts import DebugAlerter +from apscheduler.schedulers.background import BackgroundScheduler from config import get_rule_hashes from config import load_configuration from config import load_rules @@ -63,6 +67,8 @@ class ElastAlerter(): should not be passed directly from a configuration file, but must be populated by config.py:load_rules instead. """ + thread_data = threading.local() + def parse_args(self, args): parser = argparse.ArgumentParser() parser.add_argument( @@ -128,6 +134,7 @@ def __init__(self, args): tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) self.conf = load_rules(self.args) + print len(self.conf['rules']), 'rules loaded' self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] @@ -140,18 +147,15 @@ def __init__(self, args): self.from_addr = self.conf.get('from_addr', 'ElastAlert') self.smtp_host = self.conf.get('smtp_host', 'localhost') self.max_aggregation = self.conf.get('max_aggregation', 10000) - self.alerts_sent = 0 - self.cumulative_hits = 0 - self.num_hits = 0 - self.num_dupes = 0 - self.current_es = None - self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.scheduler = BackgroundScheduler() self.string_multi_field_name = self.conf.get('string_multi_field_name', False) self.add_metadata_alert = self.conf.get('add_metadata_alert', False) @@ -299,7 +303,8 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, size=1, body=query, + _source_include=[timestamp_field], ignore_unavailable=True) except ElasticsearchException as e: self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query}) return '1969-12-30T00:00:00Z' @@ -381,9 +386,9 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): try: if scroll: - res = self.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) + res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) else: - res = self.current_es.search( + res = self.thread_data.current_es.search( scroll=scroll_keepalive, index=index, size=rule.get('max_query_size', self.max_query_size), @@ -391,7 +396,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) - self.total_hits = int(res['hits']['total']) + self.thread_data.total_hits = int(res['hits']['total']) if len(res.get('_shards', {}).get('failures', [])) > 0: try: @@ -411,16 +416,16 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): self.handle_error('Error running query: %s' % (e), {'rule': rule['name'], 'query': query}) return None hits = res['hits']['hits'] - self.num_hits += len(hits) + self.thread_data.num_hits += len(hits) lt = rule.get('use_local_time') status_log = "Queried rule %s from %s to %s: %s / %s hits" % ( rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), - self.num_hits, + self.thread_data.num_hits, len(hits) ) - if self.total_hits > rule.get('max_query_size', self.max_query_size): + if self.thread_data.total_hits > rule.get('max_query_size', self.max_query_size): elastalert_logger.info("%s (scrolling..)" % status_log) rule['scroll_id'] = res['_scroll_id'] else: @@ -454,7 +459,7 @@ def get_hits_count(self, rule, starttime, endtime, index): ) try: - res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) + res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -463,7 +468,7 @@ def get_hits_count(self, rule, starttime, endtime, index): self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query}) return None - self.num_hits += res['count'] + self.thread_data.num_hits += res['count'] lt = rule.get('use_local_time') elastalert_logger.info( "Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']) @@ -509,7 +514,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.search( index=index, doc_type=rule['doc_type'], body=query, @@ -517,7 +522,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, doc_type=rule['doc_type'], + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -532,7 +538,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non buckets = res['aggregations']['filtered']['counts']['buckets'] else: buckets = res['aggregations']['counts']['buckets'] - self.num_hits += len(buckets) + self.thread_data.num_hits += len(buckets) lt = rule.get('use_local_time') elastalert_logger.info( 'Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)) @@ -555,7 +561,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field']) try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.search( index=index, doc_type=rule.get('doc_type'), body=query, @@ -563,7 +569,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, doc_type=rule.get('doc_type'), + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -575,7 +582,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ payload = res['aggregations']['filtered'] else: payload = res['aggregations'] - self.num_hits += res['hits']['total'] + self.thread_data.num_hits += res['hits']['total'] return {endtime: payload} def remove_duplicate_events(self, data, rule): @@ -629,7 +636,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if data: old_len = len(data) data = self.remove_duplicate_events(data, rule) - self.num_dupes += old_len - len(data) + self.thread_data.num_dupes += old_len - len(data) # There was an exception while querying if data is None: @@ -645,7 +652,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): rule_inst.add_data(data) try: - if rule.get('scroll_id') and self.num_hits < self.total_hits: + if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits: self.run_query(rule, start, end, scroll=True) except RuntimeError: # It's possible to scroll far enough to hit max recursive depth @@ -689,7 +696,6 @@ def get_starttime(self, rule): def set_starttime(self, rule, endtime): """ Given a rule and an endtime, sets the appropriate starttime for it. """ - # This means we are starting fresh if 'starttime' not in rule: if not rule.get('scan_entire_timeframe'): @@ -844,8 +850,7 @@ def run_rule(self, rule, endtime, starttime=None): """ run_start = time.time() - self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) + self.thread_data.current_es = elasticsearch_client(rule) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -866,9 +871,9 @@ def run_rule(self, rule, endtime, starttime=None): return 0 # Run the rule. If querying over a large time period, split it up into segments - self.num_hits = 0 - self.num_dupes = 0 - self.cumulative_hits = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] @@ -877,15 +882,15 @@ def run_rule(self, rule, endtime, starttime=None): tmp_endtime = tmp_endtime + segment_size if not self.run_query(rule, rule['starttime'], tmp_endtime): return 0 - self.cumulative_hits += self.num_hits - self.num_hits = 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits + self.thread_data.num_hits = 0 rule['starttime'] = tmp_endtime rule['type'].garbage_collect(tmp_endtime) if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: self.run_query(rule, tmp_endtime, endtime) - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] return 0 @@ -894,14 +899,14 @@ def run_rule(self, rule, endtime, starttime=None): else: if not self.run_query(rule, rule['starttime'], endtime): return 0 - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits rule['type'].garbage_collect(endtime) # Process any new matches num_matches = len(rule['type'].matches) while rule['type'].matches: match = rule['type'].matches.pop(0) - match['num_hits'] = self.cumulative_hits + match['num_hits'] = self.thread_data.cumulative_hits match['num_matches'] = num_matches # If realert is set, silence the rule for that duration @@ -947,7 +952,7 @@ def run_rule(self, rule, endtime, starttime=None): 'endtime': endtime, 'starttime': rule['original_starttime'], 'matches': num_matches, - 'hits': max(self.num_hits, self.cumulative_hits), + 'hits': max(self.thread_data.num_hits, self.thread_data.cumulative_hits), '@timestamp': ts_now(), 'time_taken': time_taken} self.writeback('elastalert_status', body) @@ -956,6 +961,9 @@ def run_rule(self, rule, endtime, starttime=None): def init_rule(self, new_rule, new=True): ''' Copies some necessary non-config state from an exiting rule to a new rule. ''' + if not new: + self.scheduler.remove_job(job_id=new_rule['name']) + try: self.modify_rule_for_ES5(new_rule) except TransportError as e: @@ -990,7 +998,9 @@ def init_rule(self, new_rule, new=True): blank_rule = {'agg_matches': [], 'aggregate_alert_time': {}, 'current_aggregate_id': {}, - 'processed_hits': {}} + 'processed_hits': {}, + 'run_every': self.run_every, + 'has_run_once': False} rule = blank_rule # Set rule to either a blank template or existing rule with same name @@ -1006,12 +1016,22 @@ def init_rule(self, new_rule, new=True): 'aggregate_alert_time', 'processed_hits', 'starttime', - 'minimum_starttime'] + 'minimum_starttime', + 'has_run_once', + 'run_every'] for prop in copy_properties: if prop not in rule: continue new_rule[prop] = rule[prop] + job = self.scheduler.add_job(self.handle_rule_execution, 'interval', + args=[new_rule], + seconds=new_rule['run_every'].total_seconds(), + id=new_rule['name'], + max_instances=1, + jitter=5) + job.modify(next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=random.randint(0, 15))) + return new_rule @staticmethod @@ -1118,14 +1138,20 @@ def start(self): except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (self.starttime)) exit(1) + + for rule in self.rules: + rule['initial_starttime'] = self.starttime self.wait_until_responsive(timeout=self.args.timeout) self.running = True elastalert_logger.info("Starting up") + self.scheduler.add_job(self.handle_pending_alerts, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_pending_alerts') + self.scheduler.add_job(self.handle_config_change, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_config_change') + self.scheduler.start() while self.running: next_run = datetime.datetime.utcnow() + self.run_every - self.run_all_rules() - # Quit after end_time has been reached if self.args.end: endtime = ts_to_dt(self.args.end) @@ -1176,53 +1202,95 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): def run_all_rules(self): """ Run each rule one time """ + self.handle_pending_alerts() + + for rule in self.rules: + self.handle_rule_execution(rule) + + self.handle_config_change() + + def handle_pending_alerts(self): + self.thread_data.alerts_sent = 0 self.send_pending_alerts() + elastalert_logger.info("Background alerts thread %s pending alerts sent at %s" % (self.thread_data.alerts_sent, + pretty_ts(ts_now()))) - next_run = datetime.datetime.utcnow() + self.run_every + def handle_config_change(self): + if not self.args.pin_rules: + self.load_rule_changes() + elastalert_logger.info("Background configuration change check run at %s" % (pretty_ts(ts_now()))) + + def handle_rule_execution(self, rule): + self.thread_data.alerts_sent = 0 + next_run = datetime.datetime.utcnow() + rule['run_every'] + # Set endtime based on the rule's delay + delay = rule.get('query_delay') + if hasattr(self.args, 'end') and self.args.end: + endtime = ts_to_dt(self.args.end) + elif delay: + endtime = ts_now() - delay + else: + endtime = ts_now() + + # Apply rules based on execution time limits + if rule.get('limit_execution'): + rule['next_starttime'] = None + rule['next_min_starttime'] = None + exec_next = croniter(rule['limit_execution']).next() + endtime_epoch = dt_to_unix(endtime) + # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time + # That means that we need to pause execution after this run + if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59: + # apscheduler requires pytz tzinfos, so don't use unix_to_dt here! + rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc) + if rule.get('limit_execution_coverage'): + rule['next_min_starttime'] = rule['next_starttime'] + if not rule['has_run_once']: + self.reset_rule_schedule(rule) + return - for rule in self.rules: - # Set endtime based on the rule's delay - delay = rule.get('query_delay') - if hasattr(self.args, 'end') and self.args.end: - endtime = ts_to_dt(self.args.end) - elif delay: - endtime = ts_now() - delay - else: - endtime = ts_now() + rule['has_run_once'] = True + try: + num_matches = self.run_rule(rule, endtime, rule.get('initial_starttime')) + except EAException as e: + self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) + except Exception as e: + self.handle_uncaught_exception(e, rule) + else: + old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) + elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," + " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), + self.thread_data.num_hits, self.thread_data.num_dupes, num_matches, + self.thread_data.alerts_sent)) + self.thread_data.alerts_sent = 0 - try: - num_matches = self.run_rule(rule, endtime, self.starttime) - except EAException as e: - self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) - except Exception as e: - self.handle_uncaught_exception(e, rule) - else: - old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) - total_hits = max(self.num_hits, self.cumulative_hits) - elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," - " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), - total_hits, self.num_dupes, num_matches, self.alerts_sent)) - self.alerts_sent = 0 - - if next_run < datetime.datetime.utcnow(): - # We were processing for longer than our refresh interval - # This can happen if --start was specified with a large time period - # or if we are running too slow to process events in real time. - logging.warning( - "Querying from %s to %s took longer than %s!" % ( - old_starttime, - pretty_ts(endtime, rule.get('use_local_time')), - self.run_every - ) + if next_run < datetime.datetime.utcnow(): + # We were processing for longer than our refresh interval + # This can happen if --start was specified with a large time period + # or if we are running too slow to process events in real time. + logging.warning( + "Querying from %s to %s took longer than %s!" % ( + old_starttime, + pretty_ts(endtime, rule.get('use_local_time')), + self.run_every ) + ) - self.remove_old_events(rule) + rule['initial_starttime'] = None - # Only force starttime once - self.starttime = None + self.remove_old_events(rule) - if not self.args.pin_rules: - self.load_rule_changes() + self.reset_rule_schedule(rule) + + def reset_rule_schedule(self, rule): + # We hit the end of a execution schedule, pause ourselves until next run + if rule.get('limit_execution') and rule['next_starttime']: + self.scheduler.modify_job(job_id=rule['name'], next_run_time=rule['next_starttime']) + # If we are preventing covering non-scheduled time periods, reset min_starttime and previous_endtime + if rule['next_min_starttime']: + rule['minimum_starttime'] = rule['next_min_starttime'] + rule['previous_endtime'] = rule['next_min_starttime'] + elastalert_logger.info('Pausing %s until next run at %s' % (rule['name'], pretty_ts(rule['next_starttime']))) def stop(self): """ Stop an ElastAlert runner that's been started """ @@ -1453,7 +1521,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): self.handle_error('Error while running alert %s: %s' % (alert.get_info()['type'], e), {'rule': rule['name']}) alert_exception = str(e) else: - self.alerts_sent += 1 + self.thread_data.alerts_sent += 1 alert_sent = True # Write the alert(s) to ES @@ -1834,6 +1902,7 @@ def handle_uncaught_exception(self, exception, rule): if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] self.disabled_rules.append(rule) + self.scheduler.pause_job(job_id=rule['name']) elastalert_logger.info('Rule %s disabled', rule['name']) if self.notify_email: self.send_notification_email(exception=exception, rule=rule) @@ -1892,7 +1961,7 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): buckets = hits_terms.values()[0] # get_hits_terms adds to num_hits, but we don't want to count these - self.num_hits -= len(buckets) + self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index d6c902514..2744e35c8 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from util import pretty_ts class BaseEnhancement(object): @@ -14,6 +15,11 @@ def process(self, match): raise NotImplementedError() +class TimeEnhancement(BaseEnhancement): + def process(self, match): + match['@timestamp'] = pretty_ts(match['@timestamp']) + + class DropMatchException(Exception): """ ElastAlert will drop a match if this exception type is raised by an enhancement """ pass diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..29cf24fbe 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -5,7 +5,7 @@ import os import dateutil.parser -import dateutil.tz +import pytz from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch @@ -112,7 +112,7 @@ def ts_to_dt(timestamp): dt = dateutil.parser.parse(timestamp) # Implicitly convert local timestamps to UTC if dt.tzinfo is None: - dt = dt.replace(tzinfo=dateutil.tz.tzutc()) + dt = dt.replace(tzinfo=pytz.utc) return dt @@ -365,6 +365,15 @@ def build_es_conn_config(conf): return parsed_conf +def pytzfy(dt): + # apscheduler requires pytz timezone objects + # This function will replace a dateutil.tz one with a pytz one + if dt.tzinfo is not None: + new_tz = pytz.timezone(dt.tzinfo.tzname('Y is this even required??')) + return dt.replace(tzinfo=new_tz) + return dt + + def parse_duration(value): """Convert ``unit=num`` spec into a ``timedelta`` object.""" unit, num = value.split('=') diff --git a/requirements-dev.txt b/requirements-dev.txt index 36daa0ebd..1cb67cb8e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +-r requirements.txt coverage flake8 pre-commit diff --git a/requirements.txt b/requirements.txt index 4f23f2ec1..418f92869 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ +apscheduler>=3.3.0 aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 +cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 elasticsearch @@ -11,11 +13,10 @@ jsonschema>=2.6.0 mock>=2.0.0 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 +python-magic>=0.4.15 PyYAML>=3.12 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 -twilio==6.0.0 thehive4py>=1.4.4 -python-magic>=0.4.15 -cffi>=1.11.5 +twilio==6.0.0 diff --git a/setup.py b/setup.py index 865d7974f..b2528e3b4 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml']}, install_requires=[ + 'apscheduler>=3.3.0' 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', diff --git a/tests/base_test.py b/tests/base_test.py index b10eb5a74..fa9018b11 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -88,9 +88,9 @@ def test_init_rule(ea): def test_query(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -98,9 +98,9 @@ def test_query(ea): def test_query_with_fields(ea): ea.rules[0]['_source_enabled'] = False - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -109,11 +109,11 @@ def test_query_with_fields(ea): def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -122,18 +122,18 @@ def test_query_with_unix(ea): def test_query_with_unixms(ea): ea.rules[0]['timestamp_type'] = 'unixms' ea.rules[0]['dt_to_ts'] = dt_to_unixms - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) def test_no_hits(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 0 @@ -142,7 +142,7 @@ def test_no_terms_hits(ea): ea.rules[0]['use_terms_query'] = True ea.rules[0]['query_key'] = 'QWERTY' ea.rules[0]['doc_type'] = 'uiop' - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_terms_data.call_count == 0 @@ -150,7 +150,7 @@ def test_no_terms_hits(ea): def test_some_hits(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -162,7 +162,7 @@ def test_some_hits_unix(ea): ea.rules[0]['ts_to_dt'] = unix_to_dt hits = generate_hits([dt_to_unix(START), dt_to_unix(END)]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = copy.deepcopy(hits) + ea.thread_data.current_es.search.return_value = copy.deepcopy(hits) ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -176,7 +176,7 @@ def _duplicate_hits_generator(timestamps, **kwargs): def test_duplicate_timestamps(ea): - ea.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') + ea.thread_data.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') ea.run_query(ea.rules[0], START, ts_to_dt('2014-01-01T00:00:00Z')) assert len(ea.rules[0]['type'].add_data.call_args_list[0][0][0]) == 3 @@ -189,7 +189,7 @@ def test_duplicate_timestamps(ea): def test_match(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -280,7 +280,7 @@ def test_match_with_module_with_agg(ea): ea.rules[0]['match_enhancements'] = [mod] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -294,7 +294,7 @@ def test_match_with_enhancements_first(ea): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) ea.rules[0]['run_enhancements_first'] = True hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch.object(ea, 'add_aggregated_alert') as add_alert: @@ -317,7 +317,7 @@ def test_agg_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10)) hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): # Aggregate first two, query over full range ea.rules[0]['aggregate_by_match_time'] = True @@ -373,7 +373,7 @@ def test_agg_not_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] match_time = ts_to_dt('2014-09-26T12:55:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -402,7 +402,7 @@ def test_agg_cron(ea): ea.max_aggregation = 1337 hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00')) alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00')) @@ -439,7 +439,7 @@ def test_agg_no_writeback_connectivity(ea): run again, that they will be passed again to add_aggregated_alert """ hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45' hits = generate_hits([hit1, hit2, hit3]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': hit1}, {'@timestamp': hit2}, @@ -453,7 +453,7 @@ def test_agg_no_writeback_connectivity(ea): {'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3}, {'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}] - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() with mock.patch('elastalert.elastalert.elasticsearch_client'): @@ -469,7 +469,7 @@ def test_agg_with_aggregation_key(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45'] match_time = ts_to_dt('2014-09-26T12:45:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -562,7 +562,7 @@ def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'☃', those=4) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] @@ -604,7 +604,7 @@ def test_silence_query_key(ea): def test_realert(ea): hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)] matches = [{'@timestamp': x} for x in hits] - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.rules[0]['realert'] = datetime.timedelta(seconds=50) ea.rules[0]['type'].matches = matches @@ -703,7 +703,7 @@ def test_count(ea): query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) start = start + ea.run_every - ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) + ea.thread_data.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -727,8 +727,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): def test_query_segmenting_reset_num_hits(ea): # Tests that num_hits gets reset every time run_query is run def assert_num_hits_reset(): - assert ea.num_hits == 0 - ea.num_hits += 10 + assert ea.thread_data.num_hits == 0 + ea.thread_data.num_hits += 10 with mock.patch.object(ea, 'run_query') as mock_run_query: mock_run_query.side_effect = assert_num_hits_reset() ea.run_rule(ea.rules[0], END, START) @@ -915,6 +915,7 @@ def test_kibana_dashboard(ea): def test_rule_changes(ea): + re = datetime.timedelta(minutes=10) ea.rule_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule2.yaml': 'DEF'} ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': []}, @@ -926,8 +927,8 @@ def test_rule_changes(ea): with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, - {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] + mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml', 'run_every': re}, + {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml', 'run_every': re}] mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1004,9 +1005,9 @@ def test_count_keys(ea): ea.rules[0]['doc_type'] = 'blah' buckets = [{'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, {'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] - ea.current_es.search.side_effect = buckets + ea.thread_data.current_es.search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) - calls = ea.current_es.search.call_args_list + calls = ea.thread_data.current_es.search.call_args_list assert calls[0][1]['search_type'] == 'count' assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5} assert counts['top_events_this'] == {'a': 10, 'b': 5} @@ -1131,7 +1132,7 @@ def mock_loop(): ea.stop() with mock.patch.object(ea, 'sleep_for', return_value=None): - with mock.patch.object(ea, 'run_all_rules') as mock_run: + with mock.patch.object(ea, 'sleep_for') as mock_run: mock_run.side_effect = mock_loop() start_thread = threading.Thread(target=ea.start) # Set as daemon to prevent a failed test from blocking exit diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..bf066122a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import datetime - import logging -import mock import os + +import mock import pytest import elastalert.elastalert @@ -87,7 +87,8 @@ def ea(): 'max_query_size': 10000, 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts, - '_source_enabled': True}] + '_source_enabled': True, + 'run_every': datetime.timedelta(seconds=15)}] conf = {'rules_folder': 'rules', 'run_every': datetime.timedelta(minutes=10), 'buffer_time': datetime.timedelta(minutes=5), @@ -103,14 +104,18 @@ def ea(): elastalert.elastalert.elasticsearch_client = mock_es_client with mock.patch('elastalert.elastalert.get_rule_hashes'): with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) + with mock.patch('elastalert.elastalert.BackgroundScheduler'): + load_conf.return_value = conf + ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() ea.writeback_es.search.return_value = {'hits': {'hits': []}} ea.writeback_es.index.return_value = {'_id': 'ABCD'} ea.current_es = mock_es_client('', '') + ea.thread_data.current_es = ea.current_es + ea.thread_data.num_hits = 0 + ea.thread_data.num_dupes = 0 return ea From c5934546b640d5882529af7ecefea5fb6614807e Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 14:37:19 -0800 Subject: [PATCH 19/74] Version 0.2.0b1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2528e3b4..0fe5b5f32 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.37', + version='0.2.0b1', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 08b42684103a6749897876161067f36044c14cc2 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 16:08:49 -0800 Subject: [PATCH 20/74] Added a message if there are no results when using count-only --- elastalert/test_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 3321d5495..0bbd10a69 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from __future__ import print_function +import argparse import copy import datetime import json @@ -13,7 +14,6 @@ import string import sys -import argparse import mock import yaml @@ -62,7 +62,6 @@ def test_file(self, conf, args): if args.stop_error: exit(1) return None - start_time = ts_now() - datetime.timedelta(days=args.days) end_time = ts_now() ts = conf.get('timestamp_field', '@timestamp') @@ -87,6 +86,7 @@ def test_file(self, conf, args): return None num_hits = len(res['hits']['hits']) if not num_hits: + print("Didn't get any results.") return [] terms = res['hits']['hits'][0]['_source'] From 2277407f278f6a1c3a504e26d50bc4089ac1253a Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 14 Nov 2018 14:38:35 +0000 Subject: [PATCH 21/74] * Repaired tests --- elastalert/loaders.py | 9 ++++++++- tests/alerts_test.py | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index c4037e997..3b7b1db4e 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -64,15 +64,18 @@ class RulesLoader(object): 'stride': alerts.StrideAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, + 'mattermost': alerts.MattermostAlerter, 'pagerduty': alerts.PagerDutyAlerter, 'exotel': alerts.ExotelAlerter, 'twilio': alerts.TwilioAlerter, 'victorops': alerts.VictorOpsAlerter, 'telegram': alerts.TelegramAlerter, + 'googlechat': alerts.GoogleChatAlerter, 'gitter': alerts.GitterAlerter, 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter + 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list @@ -108,6 +111,10 @@ def load(self, conf, args=None): for rule_file in rule_files: try: rule = self.load_configuration(rule_file, conf, args) + # A rule failed to load, don't try to process it + if not rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue # By setting "is_enabled: False" in rule file, a rule is easily disabled if 'is_enabled' in rule and not rule['is_enabled']: continue diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 8653e5aec..8169601da 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1022,7 +1022,8 @@ def test_slack_uses_custom_timeout(): 'alert': [], 'slack_timeout': 20 } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1155,7 +1156,8 @@ def test_slack_uses_list_of_custom_slack_channel(): 'slack_channel_override': ['#test-alert', '#test-alert2'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', From adea2dd8d9e51e27b9187cc7a1148ab2ccaa08af Mon Sep 17 00:00:00 2001 From: ryansaunders Date: Wed, 28 Nov 2018 14:02:01 +0000 Subject: [PATCH 22/74] Updated the writeback suffix to include customer id --- elastalert/elastalert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 78d7525bc..94892d28a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -200,7 +200,7 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_writeback_index(self, doc_type, rule=None): + def get_writeback_index(self, doc_type, rule=None, match_body=None): writeback_index = self.writeback_index if rule is None or 'writeback_suffix' not in rule: if self.is_atleastsix(): @@ -213,8 +213,8 @@ def get_writeback_index(self, doc_type, rule=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix']) - writeback_index += '_' + suffix + suffix = '_' + "{[customers][0][uuid]}_{}".format(match_body, datetime.datetime.utcnow().strftime(rule['writeback_suffix'])) + writeback_index += suffix return writeback_index @@ -1415,7 +1415,7 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No return body def writeback(self, doc_type, body, rule=None): - writeback_index = self.get_writeback_index(doc_type, rule) + writeback_index = self.get_writeback_index(doc_type, rule, body) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: From e5bb26f7a865fbfc658ba536431e2330dd8e1d5d Mon Sep 17 00:00:00 2001 From: ryansaunders Date: Wed, 28 Nov 2018 14:32:57 +0000 Subject: [PATCH 23/74] writeback suffix is formatted with content from match body --- elastalert/elastalert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 94892d28a..257307b0d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,8 +213,8 @@ def get_writeback_index(self, doc_type, rule=None, match_body=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = '_' + "{[customers][0][uuid]}_{}".format(match_body, datetime.datetime.utcnow().strftime(rule['writeback_suffix'])) - writeback_index += suffix + suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix'].format(match_body)) + writeback_index += '_' + suffix return writeback_index From 80f27e29e8dbbec5eee9c5c61a099b5a034f772d Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 29 Nov 2018 08:24:11 +0000 Subject: [PATCH 24/74] Added try-except in case suffix key is not in match_body --- elastalert/elastalert.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 257307b0d..3b6df171f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,8 +213,13 @@ def get_writeback_index(self, doc_type, rule=None, match_body=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix'].format(match_body)) - writeback_index += '_' + suffix + try: + suffix = rule['writeback_suffix'].format(match_body or {}) + suffix = datetime.datetime.utcnow().strftime(suffix) + writeback_index += '_' + suffix + except KeyError as e: + elastalert_logger.critical('Failed to add suffix. Unknown key %s' % str(e)) + pass return writeback_index From 508bf6dde4ae7884965c9f1513f134049fe24037 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 29 Nov 2018 15:36:04 +0000 Subject: [PATCH 25/74] Correctly check if template exists and delete --- elastalert/create_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b244693f7..c03922e83 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -228,9 +228,9 @@ def main(): if es_index.exists(index): print('Index ' + index + ' already exists. Skipping index creation.') return None - if es_index.exists_template(index): - print('Template ' + index + ' already exists. Deleting in preparation for creating indexes.') - es_index.delete_template(index) + if es_index.exists_template('elastalert'): + print('Template elastalert already exists. Deleting in preparation for creating indexes.') + es_index.delete_template('elastalert') # (Re-)Create indices. if elasticversion > 5: From 6c540e363344566fca0a86356788ff9e75e9e98c Mon Sep 17 00:00:00 2001 From: Sean Kang Date: Fri, 30 Nov 2018 17:17:29 +1300 Subject: [PATCH 26/74] Add a missing comma to fix dependency error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0fe5b5f32..c86885b39 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml']}, install_requires=[ - 'apscheduler>=3.3.0' + 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', From a6f92e3a6f921af5c9922d8629cf01c4e647c92c Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 13 Dec 2018 16:09:02 +0000 Subject: [PATCH 27/74] Completed merge remote-tracking branch 'github/beta' into beta/loader --- elastalert/loaders.py | 2 ++ elastalert/test_rule.py | 1 + elastalert/util.py | 1 + 3 files changed, 4 insertions(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 3b7b1db4e..a1cadd342 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -246,6 +246,8 @@ def load_options(self, rule, conf, filename, args=None): rule['query_delay'] = datetime.timedelta(**rule['query_delay']) if 'buffer_time' in rule: rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) if 'bucket_interval' in rule: rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) if 'exponential_realert' in rule: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index dba9ae785..350f12d18 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -370,6 +370,7 @@ def run_rule_test(self): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': {'hours': 24}, 'old_query_limit': {'weeks': 1}, diff --git a/elastalert/util.py b/elastalert/util.py index ef077c223..dcffc1857 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -20,6 +20,7 @@ def get_module(module_name): """ Loads a module and returns a specific object. module_name should 'module.file.object'. Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) try: module_path, module_class = module_name.rsplit('.', 1) base_module = __import__(module_path, globals(), locals(), [module_class]) From 59b697563f2460ff4067da463bf4611d5f7ed8b2 Mon Sep 17 00:00:00 2001 From: Thomas Morledge Date: Tue, 8 Jan 2019 16:09:40 +0000 Subject: [PATCH 28/74] Changed return codes for test rule --- elastalert/test_rule.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 784d7e8a0..8400c89be 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -26,10 +26,19 @@ from elastalert.util import lookup_es_key from elastalert.util import ts_now from elastalert.util import ts_to_dt +from elastalert.util import EAException logging.getLogger().setLevel(logging.INFO) logging.getLogger('elasticsearch').setLevel(logging.WARNING) +""" +Error Codes: + 1: Error connecting to ElasticSearch + 2: Error querying ElasticSearch + 3: Invalid Rule + 4: Missing/invalid timestamp +""" + def print_terms(terms, parent): """ Prints a list of flattened dictionary keys """ @@ -56,6 +65,11 @@ def test_file(self, conf, args): try: ElastAlerter.modify_rule_for_ES5(conf) + except EAException as ea: + print('Invalid filter provided:', str(ea), file=sys.stderr) + if args.stop_error: + exit(3) + return None except Exception as e: print("Error connecting to ElasticSearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -82,7 +96,7 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(3) return None num_hits = len(res['hits']['hits']) if not num_hits: @@ -108,7 +122,7 @@ def test_file(self, conf, args): print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = res['count'] @@ -152,7 +166,7 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = len(res['hits']['hits']) @@ -240,7 +254,7 @@ def run_elastalert(self, rule, conf, args): except KeyError as e: print("All documents must have a timestamp and _id: %s" % (e), file=sys.stderr) if args.stop_error: - exit(1) + exit(4) return None # Create mock _id for documents if it's missing @@ -264,7 +278,7 @@ def get_id(): endtime = ts_to_dt(args.end) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.end)) - exit(1) + exit(4) else: endtime = ts_now() if args.start: @@ -272,7 +286,7 @@ def get_id(): starttime = ts_to_dt(args.start) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.start)) - exit(1) + exit(4) else: starttime = endtime - datetime.timedelta(days=args.days) @@ -318,7 +332,7 @@ def get_id(): if call[0][0] == 'elastalert_error': errors = True if errors and args.stop_error: - exit(1) + exit(2) def load_conf(self, rules, args): """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), From 74da1c7e328d65eaa14f79091ccd52f276189e7d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 7 Jun 2019 15:40:44 -0700 Subject: [PATCH 29/74] Python3 supprt --- elastalert/__init__.py | 11 +- elastalert/alerts.py | 134 +++++++------ elastalert/config.py | 23 +-- elastalert/create_index.py | 25 ++- elastalert/elastalert.py | 93 ++++----- elastalert/enhancements.py | 2 +- elastalert/kibana.py | 341 +++++++++++++++++---------------- elastalert/loaders.py | 55 +++--- elastalert/opsgenie.py | 22 ++- elastalert/rule_from_kibana.py | 11 +- elastalert/ruletypes.py | 65 +++---- elastalert/test_rule.py | 9 +- elastalert/util.py | 10 +- setup.py | 2 +- tests/alerts_test.py | 114 +++++------ tests/base_test.py | 25 +-- tests/conftest.py | 2 +- tests/create_index_test.py | 12 +- tests/elasticsearch_test.py | 6 +- tests/loaders_test.py | 19 +- tox.ini | 2 +- 21 files changed, 483 insertions(+), 500 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 90b14126e..05d9ef7ff 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- import copy import logging -from elasticsearch import Elasticsearch, RequestsHttpConnection -from elasticsearch.client import query_params, _make_path + +from elasticsearch import Elasticsearch +from elasticsearch import RequestsHttpConnection +from elasticsearch.client import _make_path +from elasticsearch.client import query_params class ElasticSearchClient(Elasticsearch): @@ -59,14 +62,14 @@ def is_atleastsixtwo(self): """ Returns True when the Elasticsearch server version >= 6.2 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) def is_atleastsixsix(self): """ Returns True when the Elasticsearch server version >= 6.6 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 6) def is_atleastseven(self): diff --git a/elastalert/alerts.py b/elastalert/alerts.py index c02dae9b1..d3ee892d4 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -12,7 +12,7 @@ import warnings from email.mime.text import MIMEText from email.utils import formatdate -from HTMLParser import HTMLParser +from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError @@ -35,13 +35,14 @@ from thehive4py.models import CustomFieldHelper from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient -from util import EAException -from util import elastalert_logger -from util import lookup_es_key -from util import pretty_ts -from util import resolve_string -from util import ts_now -from util import ts_to_dt + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import pretty_ts +from .util import resolve_string +from .util import ts_now +from .util import ts_to_dt class DateTimeEncoder(json.JSONEncoder): @@ -65,7 +66,7 @@ def _ensure_new_line(self): def _add_custom_alert_text(self): missing = self.rule.get('alert_missing_value', '') - alert_text = unicode(self.rule.get('alert_text', '')) + alert_text = str(self.rule.get('alert_text', '')) if 'alert_text_args' in self.rule: alert_text_args = self.rule.get('alert_text_args') alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args] @@ -83,7 +84,7 @@ def _add_custom_alert_text(self): alert_text = alert_text.format(*alert_text_values) elif 'alert_text_kw' in self.rule: kw = {} - for name, kw_name in self.rule.get('alert_text_kw').items(): + for name, kw_name in list(self.rule.get('alert_text_kw').items()): val = lookup_es_key(self.match, name) # Support referencing other top-level rule properties @@ -101,10 +102,10 @@ def _add_rule_text(self): self.text += self.rule['type'].get_match_str(self.match) def _add_top_counts(self): - for key, counts in self.match.items(): + for key, counts in list(self.match.items()): if key.startswith('top_events_'): self.text += '%s:\n' % (key[11:]) - top_events = counts.items() + top_events = list(counts.items()) if not top_events: self.text += 'No events found.\n' @@ -116,12 +117,12 @@ def _add_top_counts(self): self.text += '\n' def _add_match_items(self): - match_items = self.match.items() + match_items = list(self.match.items()) match_items.sort(key=lambda x: x[0]) for key, value in match_items: if key.startswith('top_events_'): continue - value_str = unicode(value) + value_str = str(value) value_str.replace('\\n', '\n') if type(value) in [list, dict]: try: @@ -157,9 +158,9 @@ def __str__(self): class JiraFormattedMatchString(BasicMatchString): def _add_match_items(self): - match_items = dict([(x, y) for x, y in self.match.items() if not x.startswith('top_events_')]) + match_items = dict([(x, y) for x, y in list(self.match.items()) if not x.startswith('top_events_')]) json_blob = self._pretty_print_as_json(match_items) - preformatted_text = u'{{code}}{0}{{code}}'.format(json_blob) + preformatted_text = '{{code}}{0}{{code}}'.format(json_blob) self.text += preformatted_text @@ -188,14 +189,14 @@ def resolve_rule_references(self, root): root[i] = self.resolve_rule_reference(item) elif type(root) == dict: # Make a copy since we may be modifying the contents of the structure we're walking - for key, value in root.copy().iteritems(): + for key, value in root.copy().items(): if type(value) == dict or type(value) == list: self.resolve_rule_references(root[key]) else: root[key] = self.resolve_rule_reference(value) def resolve_rule_reference(self, value): - strValue = unicode(value) + strValue = str(value) if strValue.startswith('$') and strValue.endswith('$') and strValue[1:-1] in self.rule: if type(value) == int: return int(self.rule[strValue[1:-1]]) @@ -227,7 +228,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - alert_subject = unicode(self.rule['alert_subject']) + alert_subject = str(self.rule['alert_subject']) alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048)) if 'alert_subject_args' in self.rule: @@ -256,7 +257,7 @@ def create_alert_body(self, matches): body = self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -286,16 +287,16 @@ def get_aggregation_summary_text(self, matches): # Maintain an aggregate count for each unique key encountered in the aggregation period for match in matches: - key_tuple = tuple([unicode(lookup_es_key(match, key)) for key in summary_table_fields]) + key_tuple = tuple([str(lookup_es_key(match, key)) for key in summary_table_fields]) if key_tuple not in match_aggregation: match_aggregation[key_tuple] = 1 else: match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1 - for keys, count in match_aggregation.iteritems(): + for keys, count in match_aggregation.items(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' text += self.rule.get('summary_prefix', '') - return unicode(text) + return str(text) def create_default_title(self, matches): return self.rule['name'] @@ -351,13 +352,13 @@ def alert(self, matches): ) fullmessage['match'] = lookup_es_key( match, self.rule['timestamp_field']) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) fullmessage['alerts'] = alerts fullmessage['rule'] = self.rule['name'] fullmessage['rule_file'] = self.rule['rule_file'] - fullmessage['matching'] = unicode(BasicMatchString(self.rule, match)) + fullmessage['matching'] = str(BasicMatchString(self.rule, match)) fullmessage['alertDate'] = datetime.datetime.now( ).strftime("%Y-%m-%d %H:%M:%S") fullmessage['body'] = self.create_alert_body(matches) @@ -396,7 +397,7 @@ def alert(self, matches): 'Alert for %s, %s at %s:' % (self.rule['name'], match[qk], lookup_es_key(match, self.rule['timestamp_field']))) else: elastalert_logger.info('Alert for %s at %s:' % (self.rule['name'], lookup_es_key(match, self.rule['timestamp_field']))) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) def get_info(self): return {'type': 'debug'} @@ -418,15 +419,15 @@ def __init__(self, *args): self.smtp_key_file = self.rule.get('smtp_key_file') self.smtp_cert_file = self.rule.get('smtp_cert_file') # Convert email to a list if it isn't already - if isinstance(self.rule['email'], basestring): + if isinstance(self.rule['email'], str): self.rule['email'] = [self.rule['email']] # If there is a cc then also convert it a list if it isn't cc = self.rule.get('cc') - if cc and isinstance(cc, basestring): + if cc and isinstance(cc, str): self.rule['cc'] = [self.rule['cc']] # If there is a bcc then also convert it to a list if it isn't bcc = self.rule.get('bcc') - if bcc and isinstance(bcc, basestring): + if bcc and isinstance(bcc, str): self.rule['bcc'] = [self.rule['bcc']] add_suffix = self.rule.get('email_add_domain') if add_suffix and not add_suffix.startswith('@'): @@ -443,7 +444,7 @@ def alert(self, matches): to_addr = self.rule['email'] if 'email_from_field' in self.rule: recipient = lookup_es_key(matches[0], self.rule['email_from_field']) - if isinstance(recipient, basestring): + if isinstance(recipient, str): if '@' in recipient: to_addr = [recipient] elif 'email_add_domain' in self.rule: @@ -453,9 +454,9 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - email_msg = MIMEText(body.encode('UTF-8'), 'html', _charset='UTF-8') + email_msg = MIMEText(body, 'html', _charset='UTF-8') else: - email_msg = MIMEText(body.encode('UTF-8'), _charset='UTF-8') + email_msg = MIMEText(body, _charset='UTF-8') email_msg['Subject'] = self.create_title(matches) email_msg['To'] = ', '.join(to_addr) email_msg['From'] = self.from_addr @@ -600,7 +601,7 @@ def __init__(self, rule): self.get_arbitrary_fields() except JIRAError as e: # JIRAError may contain HTML, pass along only first 1024 chars - raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])), None, sys.exc_info()[2] + raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])).with_traceback(sys.exc_info()[2]) self.set_priority() @@ -609,7 +610,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, self.priority_ids.keys())) + logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -703,7 +704,7 @@ def get_arbitrary_fields(self): # Clear jira_args self.reset_jira_args() - for jira_field, value in self.rule.iteritems(): + for jira_field, value in self.rule.items(): # If we find a field that is not covered by the set that we are aware of, it means it is either: # 1. A built-in supported field in JIRA that we don't have on our radar # 2. A custom field that a JIRA admin has configured @@ -759,7 +760,7 @@ def find_existing_ticket(self, matches): return issues[0] def comment_on_ticket(self, ticket, match): - text = unicode(JiraFormattedMatchString(self.rule, match)) + text = str(JiraFormattedMatchString(self.rule, match)) timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field'])) comment = "This alert was triggered again at %s\n%s" % (timestamp, text) self.client.add_comment(ticket, comment) @@ -834,7 +835,7 @@ def alert(self, matches): "Exception encountered when trying to add '{0}' as a watcher. Does the user exist?\n{1}" .format( watcher, ex - )), None, sys.exc_info()[2] + )).with_traceback(sys.exc_info()[2]) except JIRAError as e: raise EAException("Error creating JIRA ticket using jira_args (%s): %s" % (self.jira_args, e)) @@ -849,7 +850,7 @@ def create_alert_body(self, matches): body += self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(JiraFormattedMatchString(self.rule, match)) + body += str(JiraFormattedMatchString(self.rule, match)) if len(matches) > 1: body += '\n----------------------------------------\n' return body @@ -857,7 +858,7 @@ def create_alert_body(self, matches): def get_aggregation_summary_text(self, matches): text = super(JiraAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'{{noformat}}{0}{{noformat}}'.format(text) + text = '{{noformat}}{0}{{noformat}}'.format(text) return text def create_default_title(self, matches, for_search=False): @@ -893,7 +894,7 @@ def __init__(self, *args): self.last_command = [] self.shell = False - if isinstance(self.rule['command'], basestring): + if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: logging.warning('Warning! You could be vulnerable to shell injection!') @@ -1059,7 +1060,7 @@ class MsTeamsAlerter(Alerter): def __init__(self, rule): super(MsTeamsAlerter, self).__init__(rule) self.ms_teams_webhook_url = self.rule['ms_teams_webhook_url'] - if isinstance(self.ms_teams_webhook_url, basestring): + if isinstance(self.ms_teams_webhook_url, str): self.ms_teams_webhook_url = [self.ms_teams_webhook_url] self.ms_teams_proxy = self.rule.get('ms_teams_proxy', None) self.ms_teams_alert_summary = self.rule.get('ms_teams_alert_summary', 'ElastAlert Message') @@ -1067,7 +1068,6 @@ def __init__(self, rule): self.ms_teams_theme_color = self.rule.get('ms_teams_theme_color', '') def format_body(self, body): - body = body.encode('UTF-8') if self.ms_teams_alert_fixed_width: body = body.replace('`', "'") body = "```{0}```".format('```\n\n```'.join(x for x in body.split('\n'))).replace('\n``````', '') @@ -1111,12 +1111,12 @@ class SlackAlerter(Alerter): def __init__(self, rule): super(SlackAlerter, self).__init__(rule) self.slack_webhook_url = self.rule['slack_webhook_url'] - if isinstance(self.slack_webhook_url, basestring): + if isinstance(self.slack_webhook_url, str): self.slack_webhook_url = [self.slack_webhook_url] self.slack_proxy = self.rule.get('slack_proxy', None) self.slack_username_override = self.rule.get('slack_username_override', 'elastalert') self.slack_channel_override = self.rule.get('slack_channel_override', '') - if isinstance(self.slack_channel_override, basestring): + if isinstance(self.slack_channel_override, str): self.slack_channel_override = [self.slack_channel_override] self.slack_title_link = self.rule.get('slack_title_link', '') self.slack_title = self.rule.get('slack_title', '') @@ -1132,7 +1132,7 @@ def __init__(self, rule): def format_body(self, body): # https://api.slack.com/docs/formatting - return body.encode('UTF-8') + return body def get_aggregation_summary_text__maximum_width(self): width = super(SlackAlerter, self).get_aggregation_summary_text__maximum_width() @@ -1142,7 +1142,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(SlackAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1226,7 +1226,7 @@ def __init__(self, rule): # HTTP config self.mattermost_webhook_url = self.rule['mattermost_webhook_url'] - if isinstance(self.mattermost_webhook_url, basestring): + if isinstance(self.mattermost_webhook_url, str): self.mattermost_webhook_url = [self.mattermost_webhook_url] self.mattermost_proxy = self.rule.get('mattermost_proxy', None) self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False) @@ -1249,7 +1249,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(MattermostAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1382,7 +1382,7 @@ def alert(self, matches): matches), 'summary': self.create_title(matches), 'custom_details': { - 'information': body.encode('UTF-8'), + 'information': body, }, }, } @@ -1397,7 +1397,7 @@ def alert(self, matches): 'incident_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'details': { - "information": body.encode('UTF-8'), + "information": body, }, } @@ -1513,7 +1513,7 @@ def alert(self, matches): if response != 200: raise EAException("Error posting to Exotel, response code is %s" % response) except RequestException: - raise EAException("Error posting to Exotel"), None, sys.exc_info()[2] + raise EAException("Error posting to Exotel").with_traceback(sys.exc_info()[2]) elastalert_logger.info("Trigger sent to Exotel") def get_info(self): @@ -1606,15 +1606,15 @@ def __init__(self, rule): self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None) def alert(self, matches): - body = u'⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) + body = '⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' if len(body) > 4095: - body = body[0:4000] + u"\n⚠ *message was cropped according to telegram limits!* ⚠" - body += u' ```' + body = body[0:4000] + "\n⚠ *message was cropped according to telegram limits!* ⚠" + body += ' ```' headers = {'content-type': 'application/json'} # set https proxy, if it was provided @@ -1649,7 +1649,7 @@ class GoogleChatAlerter(Alerter): def __init__(self, rule): super(GoogleChatAlerter, self).__init__(rule) self.googlechat_webhook_url = self.rule['googlechat_webhook_url'] - if isinstance(self.googlechat_webhook_url, basestring): + if isinstance(self.googlechat_webhook_url, str): self.googlechat_webhook_url = [self.googlechat_webhook_url] self.googlechat_format = self.rule.get('googlechat_format', 'basic') self.googlechat_header_title = self.rule.get('googlechat_header_title', None) @@ -1689,7 +1689,7 @@ def create_card(self, matches): card = {"cards": [{ "sections": [{ "widgets": [ - {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} + {"textParagraph": {"text": self.create_alert_body(matches)}} ]} ]} ]} @@ -1707,7 +1707,6 @@ def create_card(self, matches): def create_basic(self, matches): body = self.create_alert_body(matches) - body = body.encode('UTF-8') return {'text': body} def alert(self, matches): @@ -1865,7 +1864,6 @@ def alert(self, matches): headers = {'content-type': 'application/json'} if self.api_key is not None: headers['Authorization'] = 'Key %s' % (self.rule['alerta_api_key']) - alerta_payload = self.get_json_payload(matches[0]) try: @@ -1924,8 +1922,8 @@ def get_json_payload(self, match): 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], - 'attributes': dict(zip(self.attributes_keys, - [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), + 'attributes': dict(list(zip(self.attributes_keys, + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values]))), 'rawData': self.create_alert_body([match]), } @@ -1942,7 +1940,7 @@ class HTTPPostAlerter(Alerter): def __init__(self, rule): super(HTTPPostAlerter, self).__init__(rule) post_url = self.rule.get('http_post_url') - if isinstance(post_url, basestring): + if isinstance(post_url, str): post_url = [post_url] self.post_url = post_url self.post_proxy = self.rule.get('http_post_proxy') @@ -1957,7 +1955,7 @@ def alert(self, matches): for match in matches: payload = match if self.post_all_values else {} payload.update(self.post_static_payload) - for post_key, es_key in self.post_payload.items(): + for post_key, es_key in list(self.post_payload.items()): payload[post_key] = lookup_es_key(match, es_key) headers = { "Content-Type": "application/json", @@ -2123,12 +2121,12 @@ def alert(self, matches): artifacts = [] for mapping in self.rule.get('hive_observable_data_mapping', []): - for observable_type, match_data_key in mapping.iteritems(): + for observable_type, match_data_key in mapping.items(): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) data_keys = match_data_keys + rule_data_keys - context_keys = context['match'].keys() + context['rule'].keys() + context_keys = list(context['match'].keys()) + list(context['rule'].keys()) if all([True if k in context_keys else False for k in data_keys]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: @@ -2141,10 +2139,10 @@ def alert(self, matches): } alert_config.update(self.rule.get('hive_alert_config', {})) - for alert_config_field, alert_config_value in alert_config.iteritems(): + for alert_config_field, alert_config_value in alert_config.items(): if alert_config_field == 'customFields': custom_fields = CustomFieldHelper() - for cf_key, cf_value in alert_config_value.iteritems(): + for cf_key, cf_value in alert_config_value.items(): try: func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) except AttributeError: @@ -2152,7 +2150,7 @@ def alert(self, matches): value = cf_value['value'].format(**context) func(cf_key, value) alert_config[alert_config_field] = custom_fields.build() - elif isinstance(alert_config_value, basestring): + elif isinstance(alert_config_value, str): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] diff --git a/elastalert/config.py b/elastalert/config.py index cc87a36d1..23dbf1d55 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -3,12 +3,13 @@ import logging import logging.config -import loaders from envparse import Env from staticconf.loader import yaml_loader -from util import EAException -from util import elastalert_logger -from util import get_module + +from . import loaders +from .util import EAException +from .util import elastalert_logger +from .util import get_module # Required global (config.yaml) configuration options @@ -46,21 +47,21 @@ def load_conf(args, defaults=None, overwrites=None): # init logging from config and set log levels according to command line options configure_logging(args, conf) - for env_var, conf_var in env_settings.items(): + for env_var, conf_var in list(env_settings.items()): val = env(env_var, None) if val is not None: conf[conf_var] = val - for key, value in (defaults.iteritems() if defaults is not None else []): + for key, value in (iter(defaults.items()) if defaults is not None else []): if key not in conf: conf[key] = value - for key, value in (overwrites.iteritems() if overwrites is not None else []): + for key, value in (iter(overwrites.items()) if overwrites is not None else []): conf[key] = value # Make sure we have all required globals - if required_globals - frozenset(conf.keys()): - raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + if required_globals - frozenset(list(conf.keys())): + raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(list(conf.keys()))))) conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) @@ -90,9 +91,9 @@ def load_conf(args, defaults=None, overwrites=None): conf['rules_loader'] = rules_loader # Make sure we have all the required globals for the loader # Make sure we have all required globals - if rules_loader.required_globals - frozenset(conf.keys()): + if rules_loader.required_globals - frozenset(list(conf.keys())): raise EAException( - '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(conf.keys())))) + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(list(conf.keys()))))) return conf diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 009310258..39e3cbb60 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function - import argparse import getpass import json @@ -10,13 +8,14 @@ import elasticsearch.helpers import yaml -from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch.exceptions import NotFoundError from envparse import Env +from .auth import Auth + env = Env(ES_USE_SSL=bool) @@ -139,7 +138,7 @@ def is_atleastsix(es_version): def is_atleastsixtwo(es_version): - major, minor = map(int, es_version.split(".")[:2]) + major, minor = list(map(int, es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) @@ -213,32 +212,32 @@ def main(): username = args.username if args.username else None password = args.password if args.password else None aws_region = args.aws_region - host = args.host if args.host else raw_input('Enter Elasticsearch host: ') - port = args.port if args.port else int(raw_input('Enter Elasticsearch port: ')) + host = args.host if args.host else input('Enter Elasticsearch host: ') + port = args.port if args.port else int(input('Enter Elasticsearch port: ')) use_ssl = (args.ssl if args.ssl is not None - else raw_input('Use SSL? t/f: ').lower() in ('t', 'true')) + else input('Use SSL? t/f: ').lower() in ('t', 'true')) if use_ssl: verify_certs = (args.verify_certs if args.verify_certs is not None - else raw_input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) + else input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) else: verify_certs = True if args.no_auth is None and username is None: - username = raw_input('Enter optional basic-auth username (or leave blank): ') + username = input('Enter optional basic-auth username (or leave blank): ') password = getpass.getpass('Enter optional basic-auth password (or leave blank): ') url_prefix = (args.url_prefix if args.url_prefix is not None - else raw_input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) + else input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) send_get_body_as = args.send_get_body_as ca_certs = None client_cert = None client_key = None - index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') + index = args.index if args.index is not None else input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' - alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + alias = args.alias if args.alias is not None else input('New alias name? (Default elastalert_alerts) ') if not alias: alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None - else raw_input('Name of existing index to copy? (Default None) ')) + else input('Name of existing index to copy? (Default None) ')) timeout = args.timeout diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 6da26cabd..98e9dc8f0 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -18,37 +18,38 @@ from socket import error import dateutil.tz -import kibana import pytz -from alerts import DebugAlerter from apscheduler.schedulers.background import BackgroundScheduler -from config import load_conf from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException from elasticsearch.exceptions import TransportError -from enhancements import DropMatchException -from ruletypes import FlatlineRule -from util import add_raw_postfix -from util import cronite_datetime_to_timestamp -from util import dt_to_ts -from util import dt_to_unix -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import lookup_es_key -from util import parse_deadline -from util import parse_duration -from util import pretty_ts -from util import replace_dots_in_field_names -from util import seconds -from util import set_es_key -from util import total_seconds -from util import ts_add -from util import ts_now -from util import ts_to_dt -from util import unix_to_dt + +from . import kibana +from .alerts import DebugAlerter +from .config import load_conf +from .enhancements import DropMatchException +from .ruletypes import FlatlineRule +from .util import add_raw_postfix +from .util import cronite_datetime_to_timestamp +from .util import dt_to_ts +from .util import dt_to_unix +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import lookup_es_key +from .util import parse_deadline +from .util import parse_duration +from .util import pretty_ts +from .util import replace_dots_in_field_names +from .util import seconds +from .util import set_es_key +from .util import total_seconds +from .util import ts_add +from .util import ts_now +from .util import ts_to_dt +from .util import unix_to_dt class ElastAlerter(object): @@ -134,7 +135,7 @@ def __init__(self, args): self.rules_loader = self.conf['rules_loader'] self.rules = self.rules_loader.load(self.conf, self.args) - print len(self.rules), 'rules loaded' + print(len(self.rules), 'rules loaded') self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] @@ -166,7 +167,7 @@ def __init__(self, args): for rule in self.rules: if not self.init_rule(rule): remove.append(rule) - map(self.rules.remove, remove) + list(map(self.rules.remove, remove)) if self.args.silence: self.silence() @@ -306,7 +307,7 @@ def process_hits(rule, hits): for hit in hits: # Merge fields and _source hit.setdefault('_source', {}) - for key, value in hit.get('fields', {}).items(): + for key, value in list(hit.get('fields', {}).items()): # Fields are returned as lists, assume any with length 1 are not arrays in _source # Except sometimes they aren't lists. This is dependent on ES version hit['_source'].setdefault(key, value[0] if type(value) is list and len(value) == 1 else value) @@ -328,11 +329,11 @@ def process_hits(rule, hits): if rule.get('compound_query_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_query_key']] - hit['_source'][rule['query_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['query_key']] = ', '.join([str(value) for value in values]) if rule.get('compound_aggregation_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_aggregation_key']] - hit['_source'][rule['aggregation_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['aggregation_key']] = ', '.join([str(value) for value in values]) processed_hits.append(hit['_source']) @@ -595,10 +596,10 @@ def remove_old_events(self, rule): buffer_time = rule.get('buffer_time', self.buffer_time) if rule.get('query_delay'): buffer_time += rule['query_delay'] - for _id, timestamp in rule['processed_hits'].iteritems(): + for _id, timestamp in rule['processed_hits'].items(): if now - timestamp > buffer_time: remove.append(_id) - map(rule['processed_hits'].pop, remove) + list(map(rule['processed_hits'].pop, remove)) def run_query(self, rule, start=None, end=None, scroll=False): """ Query for the rule and pass all of the results to the RuleType instance. @@ -772,7 +773,7 @@ def get_query_key_value(self, rule, match): # get the value for the match's query_key (or none) to form the key used for the silence_cache. # Flatline ruletype sets "key" instead of the actual query_key if isinstance(rule['type'], FlatlineRule) and 'key' in match: - return unicode(match['key']) + return str(match['key']) return self.get_named_key_value(rule, match, 'query_key') def get_aggregation_key_value(self, rule, match): @@ -787,7 +788,7 @@ def get_named_key_value(self, rule, match, key_name): if key_value is not None: # Only do the unicode conversion if we actually found something) # otherwise we might transform None --> 'None' - key_value = unicode(key_value) + key_value = str(key_value) except KeyError: # Some matches may not have the specified key # use a special token for these @@ -1051,7 +1052,7 @@ def load_rule_changes(self): new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes - for rule_file, hash_value in self.rule_hashes.iteritems(): + for rule_file, hash_value in self.rule_hashes.items(): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) @@ -1227,7 +1228,7 @@ def handle_rule_execution(self, rule): if rule.get('limit_execution'): rule['next_starttime'] = None rule['next_min_starttime'] = None - exec_next = croniter(rule['limit_execution']).next() + exec_next = next(croniter(rule['limit_execution'])) endtime_epoch = dt_to_unix(endtime) # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time # That means that we need to pause execution after this run @@ -1383,7 +1384,7 @@ def get_dashboard(self, rule, db_name): # TODO use doc_type = _doc res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: - raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] + raise EAException("Error querying for dashboard: %s" % (e)).with_traceback(sys.exc_info()[2]) if res['hits']['hits']: return json.loads(res['hits']['hits'][0]['_source']['dashboard']) @@ -1485,7 +1486,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): try: enhancement.process(match) valid_matches.append(match) - except DropMatchException as e: + except DropMatchException: pass except EAException as e: self.handle_error("Error running match enhancement: %s" % (e), {'rule': rule['name']}) @@ -1560,7 +1561,7 @@ def writeback(self, doc_type, body, rule=None, match_body=None): else: writeback_body = body - for key in writeback_body.keys(): + for key in list(writeback_body.keys()): # Convert any datetime objects to timestamps if isinstance(writeback_body[key], datetime.datetime): writeback_body[key] = dt_to_ts(writeback_body[key]) @@ -1649,7 +1650,7 @@ def send_pending_alerts(self): self.alert([match_body], rule, alert_time=alert_time, retried=retried) if rule['current_aggregate_id']: - for qk, agg_id in rule['current_aggregate_id'].iteritems(): + for qk, agg_id in rule['current_aggregate_id'].items(): if agg_id == _id: rule['current_aggregate_id'].pop(qk) break @@ -1666,7 +1667,7 @@ def send_pending_alerts(self): # Send in memory aggregated alerts for rule in self.rules: if rule['agg_matches']: - for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].iteritems(): + for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].items(): if ts_now() > aggregate_alert_time: alertable_matches = [ agg_match @@ -1877,7 +1878,7 @@ def is_silenced(self, rule_name): if res['hits']['hits']: until_ts = res['hits']['hits'][0]['_source']['until'] exponent = res['hits']['hits'][0]['_source'].get('exponent', 0) - if rule_name not in self.silence_cache.keys(): + if rule_name not in list(self.silence_cache.keys()): self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent) else: self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1]) @@ -1925,13 +1926,13 @@ def send_notification_email(self, text='', exception=None, rule=None, subject=No tb = traceback.format_exc() email_body += tb - if isinstance(self.notify_email, basestring): + if isinstance(self.notify_email, str): self.notify_email = [self.notify_email] email = MIMEText(email_body) email['Subject'] = subject if subject else 'ElastAlert notification' recipients = self.notify_email if rule and rule.get('notify_email'): - if isinstance(rule['notify_email'], basestring): + if isinstance(rule['notify_email'], str): rule['notify_email'] = [rule['notify_email']] recipients = recipients + rule['notify_email'] recipients = list(set(recipients)) @@ -1958,14 +1959,14 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): if hits_terms is None: top_events_count = {} else: - buckets = hits_terms.values()[0] + buckets = list(hits_terms.values())[0] # get_hits_terms adds to num_hits, but we don't want to count these self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] - counts = terms.items() + counts = list(terms.items()) counts.sort(key=lambda x: x[1], reverse=True) top_events_count = dict(counts[:number]) diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index 2744e35c8..6cc1cdd57 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from util import pretty_ts +from .util import pretty_ts class BaseEnhancement(object): diff --git a/elastalert/kibana.py b/elastalert/kibana.py index 2cd557bff..de690494e 100644 --- a/elastalert/kibana.py +++ b/elastalert/kibana.py @@ -1,173 +1,176 @@ # -*- coding: utf-8 -*- +# flake8: noqa import os.path -import urllib +import urllib.error +import urllib.parse +import urllib.request -from util import EAException +from .util import EAException dashboard_temp = {'editable': True, - u'failover': False, - u'index': {u'default': u'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', - u'interval': u'none', - u'pattern': u'', - u'warm_fields': True}, - u'loader': {u'hide': False, - u'load_elasticsearch': True, - u'load_elasticsearch_size': 20, - u'load_gist': True, - u'load_local': True, - u'save_default': True, - u'save_elasticsearch': True, - u'save_gist': False, - u'save_local': True, - u'save_temp': True, - u'save_temp_ttl': u'30d', - u'save_temp_ttl_enable': True}, - u'nav': [{u'collapse': False, - u'enable': True, - u'filter_id': 0, - u'notice': False, - u'now': False, - u'refresh_intervals': [u'5s', - u'10s', - u'30s', - u'1m', - u'5m', - u'15m', - u'30m', - u'1h', - u'2h', - u'1d'], - u'status': u'Stable', - u'time_options': [u'5m', - u'15m', - u'1h', - u'6h', - u'12h', - u'24h', - u'2d', - u'7d', - u'30d'], - u'timefield': u'@timestamp', - u'type': u'timepicker'}], - u'panel_hints': True, - u'pulldowns': [{u'collapse': False, - u'enable': True, - u'notice': True, - u'type': u'filtering'}], - u'refresh': False, - u'rows': [{u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'annotate': {u'enable': False, - u'field': u'_type', - u'query': u'*', - u'size': 20, - u'sort': [u'_score', u'desc']}, - u'auto_int': True, - u'bars': True, - u'derivative': False, - u'editable': True, - u'fill': 3, - u'grid': {u'max': None, u'min': 0}, - u'group': [u'default'], - u'interactive': True, - u'interval': u'1m', - u'intervals': [u'auto', - u'1s', - u'1m', - u'5m', - u'10m', - u'30m', - u'1h', - u'3h', - u'12h', - u'1d', - u'1w', - u'1M', - u'1y'], - u'legend': True, - u'legend_counts': True, - u'lines': False, - u'linewidth': 3, - u'mode': u'count', - u'options': True, - u'percentage': False, - u'pointradius': 5, - u'points': False, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'resolution': 100, - u'scale': 1, - u'show_query': True, - u'span': 12, - u'spyable': True, - u'stack': True, - u'time_field': u'@timestamp', - u'timezone': u'browser', - u'title': u'Events over time', - u'tooltip': {u'query_as_alias': True, - u'value_type': u'cumulative'}, - u'type': u'histogram', - u'value_field': None, - u'x-axis': True, - u'y-axis': True, - u'y_format': u'none', - u'zerofill': True, - u'zoomlinks': True}], - u'title': u'Graph'}, - {u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'all_fields': False, - u'editable': True, - u'error': False, - u'field_list': True, - u'fields': [], - u'group': [u'default'], - u'header': True, - u'highlight': [], - u'localTime': True, - u'normTimes': True, - u'offset': 0, - u'overflow': u'min-height', - u'pages': 5, - u'paging': True, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'size': 100, - u'sort': [u'@timestamp', u'desc'], - u'sortable': True, - u'span': 12, - u'spyable': True, - u'status': u'Stable', - u'style': {u'font-size': u'9pt'}, - u'timeField': u'@timestamp', - u'title': u'All events', - u'trimFactor': 300, - u'type': u'table'}], - u'title': u'Events'}], - u'services': {u'filter': {u'ids': [0], - u'list': {u'0': {u'active': True, - u'alias': u'', - u'field': u'@timestamp', - u'from': u'now-24h', - u'id': 0, - u'mandate': u'must', - u'to': u'now', - u'type': u'time'}}}, - u'query': {u'ids': [0], - u'list': {u'0': {u'alias': u'', - u'color': u'#7EB26D', - u'enable': True, - u'id': 0, - u'pin': False, - u'query': u'', - u'type': u'lucene'}}}}, - u'style': u'dark', - u'title': u'ElastAlert Alert Dashboard'} + 'failover': False, + 'index': {'default': 'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', + 'interval': 'none', + 'pattern': '', + 'warm_fields': True}, + 'loader': {'hide': False, + 'load_elasticsearch': True, + 'load_elasticsearch_size': 20, + 'load_gist': True, + 'load_local': True, + 'save_default': True, + 'save_elasticsearch': True, + 'save_gist': False, + 'save_local': True, + 'save_temp': True, + 'save_temp_ttl': '30d', + 'save_temp_ttl_enable': True}, + 'nav': [{'collapse': False, + 'enable': True, + 'filter_id': 0, + 'notice': False, + 'now': False, + 'refresh_intervals': ['5s', + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d'], + 'status': 'Stable', + 'time_options': ['5m', + '15m', + '1h', + '6h', + '12h', + '24h', + '2d', + '7d', + '30d'], + 'timefield': '@timestamp', + 'type': 'timepicker'}], + 'panel_hints': True, + 'pulldowns': [{'collapse': False, + 'enable': True, + 'notice': True, + 'type': 'filtering'}], + 'refresh': False, + 'rows': [{'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'annotate': {'enable': False, + 'field': '_type', + 'query': '*', + 'size': 20, + 'sort': ['_score', 'desc']}, + 'auto_int': True, + 'bars': True, + 'derivative': False, + 'editable': True, + 'fill': 3, + 'grid': {'max': None, 'min': 0}, + 'group': ['default'], + 'interactive': True, + 'interval': '1m', + 'intervals': ['auto', + '1s', + '1m', + '5m', + '10m', + '30m', + '1h', + '3h', + '12h', + '1d', + '1w', + '1M', + '1y'], + 'legend': True, + 'legend_counts': True, + 'lines': False, + 'linewidth': 3, + 'mode': 'count', + 'options': True, + 'percentage': False, + 'pointradius': 5, + 'points': False, + 'queries': {'ids': [0], 'mode': 'all'}, + 'resolution': 100, + 'scale': 1, + 'show_query': True, + 'span': 12, + 'spyable': True, + 'stack': True, + 'time_field': '@timestamp', + 'timezone': 'browser', + 'title': 'Events over time', + 'tooltip': {'query_as_alias': True, + 'value_type': 'cumulative'}, + 'type': 'histogram', + 'value_field': None, + 'x-axis': True, + 'y-axis': True, + 'y_format': 'none', + 'zerofill': True, + 'zoomlinks': True}], + 'title': 'Graph'}, + {'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'all_fields': False, + 'editable': True, + 'error': False, + 'field_list': True, + 'fields': [], + 'group': ['default'], + 'header': True, + 'highlight': [], + 'localTime': True, + 'normTimes': True, + 'offset': 0, + 'overflow': 'min-height', + 'pages': 5, + 'paging': True, + 'queries': {'ids': [0], 'mode': 'all'}, + 'size': 100, + 'sort': ['@timestamp', 'desc'], + 'sortable': True, + 'span': 12, + 'spyable': True, + 'status': 'Stable', + 'style': {'font-size': '9pt'}, + 'timeField': '@timestamp', + 'title': 'All events', + 'trimFactor': 300, + 'type': 'table'}], + 'title': 'Events'}], + 'services': {'filter': {'ids': [0], + 'list': {'0': {'active': True, + 'alias': '', + 'field': '@timestamp', + 'from': 'now-24h', + 'id': 0, + 'mandate': 'must', + 'to': 'now', + 'type': 'time'}}}, + 'query': {'ids': [0], + 'list': {'0': {'alias': '', + 'color': '#7EB26D', + 'enable': True, + 'id': 0, + 'pin': False, + 'query': '', + 'type': 'lucene'}}}}, + 'style': 'dark', + 'title': 'ElastAlert Alert Dashboard'} kibana4_time_temp = "(refreshInterval:(display:Off,section:0,value:0),time:(from:'%s',mode:absolute,to:'%s'))" @@ -213,9 +216,9 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = es_filter['query_string']['query'] elif 'term' in es_filter: kibana_filter['type'] = 'field' - f_field, f_query = es_filter['term'].items()[0] + f_field, f_query = list(es_filter['term'].items())[0] # Wrap query in quotes, otherwise certain characters cause Kibana to throw errors - if isinstance(f_query, basestring): + if isinstance(f_query, str): f_query = '"%s"' % (f_query.replace('"', '\\"')) if isinstance(f_query, list): # Escape quotes @@ -228,7 +231,7 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = f_query elif 'range' in es_filter: kibana_filter['type'] = 'range' - f_field, f_range = es_filter['range'].items()[0] + f_field, f_range = list(es_filter['range'].items())[0] kibana_filter['field'] = f_field kibana_filter.update(f_range) else: @@ -250,7 +253,7 @@ def filters_from_dashboard(db): filters = db['services']['filter']['list'] config_filters = [] or_filters = [] - for filter in filters.values(): + for filter in list(filters.values()): filter_type = filter['type'] if filter_type == 'time': continue @@ -281,5 +284,5 @@ def filters_from_dashboard(db): def kibana4_dashboard_link(dashboard, starttime, endtime): dashboard = os.path.expandvars(dashboard) time_settings = kibana4_time_temp % (starttime, endtime) - time_settings = urllib.quote(time_settings) + time_settings = urllib.parse.quote(time_settings) return "%s?_g=%s" % (dashboard, time_settings) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index a1cadd342..db14a3a52 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -6,24 +6,25 @@ import os import sys -import alerts -import enhancements import jsonschema -import ruletypes import yaml import yaml.scanner -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import get_module -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms -from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt + +from . import alerts +from . import enhancements +from . import ruletypes +from .opsgenie import OpsGenieAlerter +from .util import dt_to_ts +from .util import dt_to_ts_with_format +from .util import dt_to_unix +from .util import dt_to_unixms +from .util import EAException +from .util import get_module +from .util import ts_to_dt +from .util import ts_to_dt_with_format +from .util import unix_to_dt +from .util import unixms_to_dt class RulesLoader(object): @@ -90,7 +91,7 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml self.rule_schema = jsonschema.Draft4Validator( - yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) self.base_config = copy.deepcopy(conf) @@ -260,7 +261,7 @@ def load_options(self, rule, conf, filename, args=None): raise EAException('Invalid time format used: %s' % e) # Set defaults, copy defaults from config.yaml - for key, val in self.base_config.items(): + for key, val in list(self.base_config.items()): rule.setdefault(key, val) rule.setdefault('name', os.path.splitext(filename)[0]) rule.setdefault('realert', datetime.timedelta(seconds=0)) @@ -317,8 +318,8 @@ def _dt_to_ts_with_format(dt): rule.setdefault('hipchat_ignore_ssl_errors', False) # Make sure we have required options - if self.required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(rule.keys())))) + if self.required_locals - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) if 'include' in rule and type(rule['include']) != list: raise EAException('include option must be a list') @@ -361,7 +362,7 @@ def _dt_to_ts_with_format(dt): es_filter = es_filter['not'] if 'query' in es_filter: es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): + if list(es_filter.keys())[0] not in ('term', 'query_string', 'range'): raise EAException( 'generate_kibana_link is incompatible with filters other than term, query_string and range.' 'Consider creating a dashboard and using use_kibana_dashboard instead.') @@ -414,13 +415,13 @@ def load_modules(self, rule, args=None): # Make sure we have required alert and type options reqs = rule['type'].required_options - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) + if reqs - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(list(rule.keys()))))) # Instantiate rule try: rule['type'] = rule['type'](rule, args) except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)).with_traceback(sys.exc_info()[2]) # Instantiate alerts only if we're not in debug mode # In debug mode alerts are not actually sent so don't bother instantiating them if not args or not args.debug: @@ -430,10 +431,10 @@ def load_alerts(self, rule, alert_field): def normalize_config(alert): """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): + if isinstance(alert, str): return alert, rule elif isinstance(alert, dict): - name, config = iter(alert.items()).next() + name, config = next(iter(list(alert.items()))) config_copy = copy.copy(rule) config_copy.update(config) # warning, this (intentionally) mutates the rule dict return name, config_copy @@ -455,12 +456,12 @@ def create_alert(alert, alert_config): alert_field = [alert_field] alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): self.alerts_order.get(a, 1)) + alert_field = sorted(alert_field, key=lambda a_b: self.alerts_order.get(a_b[0], 1)) # Convert all alerts into Alerter objects alert_field = [create_alert(a, b) for a, b in alert_field] except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)).with_traceback(sys.exc_info()[2]) return alert_field @@ -530,7 +531,7 @@ def get_import_rule(self, rule): def get_rule_file_hash(self, rule_file): rule_file_hash = '' if os.path.exists(rule_file): - with open(rule_file) as fh: + with open(rule_file, 'rb') as fh: rule_file_hash = hashlib.sha1(fh.read()).digest() for import_rule_file in self.import_rules.get(rule_file, []): rule_file_hash += self.get_rule_file_hash(import_rule_file) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 6a6e7fda7..8f58b0b26 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import json import logging + import requests -from alerts import Alerter -from alerts import BasicMatchString -from util import EAException -from util import elastalert_logger -from util import lookup_es_key + +from .alerts import Alerter +from .alerts import BasicMatchString +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key class OpsGenieAlerter(Alerter): @@ -35,11 +37,11 @@ def __init__(self, *args): def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: formated_responders = list() - responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) - responders_values = dict((k, v) for k, v in responders_values.iteritems() if v) + responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.items()) + responders_values = dict((k, v) for k, v in responders_values.items() if v) for responder in responders: - responder = unicode(responder) + responder = str(responder) try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: @@ -60,7 +62,7 @@ def _fill_responders(self, responders, type_): def alert(self, matches): body = '' for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -131,7 +133,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - opsgenie_subject = unicode(self.rule['opsgenie_subject']) + opsgenie_subject = str(self.rule['opsgenie_subject']) if self.opsgenie_subject_args: opsgenie_subject_values = [lookup_es_key(matches[0], arg) for arg in self.opsgenie_subject_args] diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index ef1392b28..4a0634954 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import json import yaml @@ -12,10 +9,10 @@ def main(): - es_host = raw_input("Elasticsearch host: ") - es_port = raw_input("Elasticsearch port: ") - db_name = raw_input("Dashboard name: ") - send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' + es_host = input("Elasticsearch host: ") + es_port = input("Elasticsearch port: ") + db_name = input("Dashboard name: ") + send_get_body_as = input("Method for querying Elasticsearch[GET]: ") or 'GET' es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as}) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index a8845ca92..2f1d2f82c 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -4,19 +4,20 @@ import sys from blist import sortedlist -from util import add_raw_postfix -from util import dt_to_ts -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import hashable -from util import lookup_es_key -from util import new_get_event_ts -from util import pretty_ts -from util import total_seconds -from util import ts_now -from util import ts_to_dt + +from .util import add_raw_postfix +from .util import dt_to_ts +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import hashable +from .util import lookup_es_key +from .util import new_get_event_ts +from .util import pretty_ts +from .util import total_seconds +from .util import ts_now +from .util import ts_to_dt class RuleType(object): @@ -205,8 +206,8 @@ def add_match(self, match): if change: extra = {'old_value': change[0], 'new_value': change[1]} - elastalert_logger.debug("Description of the changed records " + str(dict(match.items() + extra.items()))) - super(ChangeRule, self).add_match(dict(match.items() + extra.items())) + elastalert_logger.debug("Description of the changed records " + str(dict(list(match.items()) + list(extra.items())))) + super(ChangeRule, self).add_match(dict(list(match.items()) + list(extra.items()))) class FrequencyRule(RuleType): @@ -224,14 +225,14 @@ def add_count_data(self, data): if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - (ts, count), = data.items() + (ts, count), = list(data.items()) event = ({self.ts_field: ts}, count) self.occurrences.setdefault('all', EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event) self.check_for_match('all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: event = ({self.ts_field: timestamp, self.rules['query_key']: bucket['key']}, bucket['doc_count']) @@ -274,10 +275,10 @@ def check_for_match(self, key, end=False): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ stale_keys = [] - for key, window in self.occurrences.iteritems(): + for key, window in self.occurrences.items(): if timestamp - lookup_es_key(window.data[-1][0], self.ts_field) > self.rules['timeframe']: stale_keys.append(key) - map(self.occurrences.pop, stale_keys) + list(map(self.occurrences.pop, stale_keys)) def get_match_str(self, match): lt = self.rules.get('use_local_time') @@ -401,11 +402,11 @@ def add_count_data(self, data): """ Add count data to the rule. Data should be of the form {ts: count}. """ if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - for ts, count in data.iteritems(): + for ts, count in data.items(): self.handle_event({self.ts_field: ts}, count, 'all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: count = bucket['doc_count'] event = {self.ts_field: timestamp, @@ -489,7 +490,7 @@ def add_match(self, match, qk): extra_info = {'spike_count': spike_count, 'reference_count': reference_count} - match = dict(match.items() + extra_info.items()) + match = dict(list(match.items()) + list(extra_info.items())) super(SpikeRule, self).add_match(match) @@ -535,7 +536,7 @@ def get_match_str(self, match): def garbage_collect(self, ts): # Windows are sized according to their newest event # This is a placeholder to accurately size windows in the absence of events - for qk in self.cur_windows.keys(): + for qk in list(self.cur_windows.keys()): # If we havn't seen this key in a long time, forget it if qk != 'all' and self.ref_windows[qk].count() == 0 and self.cur_windows[qk].count() == 0: self.cur_windows.pop(qk) @@ -608,7 +609,7 @@ def garbage_collect(self, ts): # We add an event with a count of zero to the EventWindow for each key. This will cause the EventWindow # to remove events that occurred more than one `timeframe` ago, and call onRemoved on them. default = ['all'] if 'query_key' not in self.rules else [] - for key in self.occurrences.keys() or default: + for key in list(self.occurrences.keys()) or default: self.occurrences.setdefault( key, EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts) @@ -650,7 +651,7 @@ def __init__(self, rule, args=None): self.get_all_terms(args) except Exception as e: # Refuse to start if we cannot get existing terms - raise EAException('Error searching for existing terms: %s' % (repr(e))), None, sys.exc_info()[2] + raise EAException('Error searching for existing terms: %s' % (repr(e))).with_traceback(sys.exc_info()[2]) def get_all_terms(self, args): """ Performs a terms aggregation for each field to get every existing term. """ @@ -731,7 +732,7 @@ def get_all_terms(self, args): time_filter[self.rules['timestamp_field']] = {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)} - for key, values in self.seen_values.iteritems(): + for key, values in self.seen_values.items(): if not values: if type(key) == tuple: # If we don't have any results, it could either be because of the absence of any baseline data @@ -879,7 +880,7 @@ def add_data(self, data): def add_terms_data(self, terms): # With terms query, len(self.fields) is always 1 and the 0'th entry is always a string field = self.fields[0] - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: if bucket['doc_count']: if bucket['key'] not in self.seen_values[field]: @@ -941,8 +942,8 @@ def check_for_match(self, key, event, gc=True): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ - for qk, terms in self.cardinality_cache.items(): - for term, last_occurence in terms.items(): + for qk, terms in list(self.cardinality_cache.items()): + for term, last_occurence in list(terms.items()): if timestamp - last_occurence > self.rules['timeframe']: self.cardinality_cache[qk].pop(term) @@ -997,7 +998,7 @@ def generate_aggregation_query(self): raise NotImplementedError() def add_aggregation_data(self, payload): - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'interval_aggs' in payload_data: self.unwrap_interval_buckets(timestamp, None, payload_data['interval_aggs']['buckets']) elif 'bucket_aggs' in payload_data: @@ -1087,7 +1088,7 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou # add compound key to payload to allow alerts to trigger for every unique occurence compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join([unicode(value) for value in compound_value]) + match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) self.add_match(match_data) @@ -1133,7 +1134,7 @@ def add_aggregation_data(self, payload): We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from the aggregation's "value" key to determine spikes from aggregations """ - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'bucket_aggs' in payload_data: self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) else: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 293f0d0f6..72032de1b 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import argparse import copy import datetime @@ -125,7 +122,7 @@ def test_file(self, conf, args): if args.formatted_output: self.formatted_output['hits'] = num_hits self.formatted_output['days'] = args.days - self.formatted_output['terms'] = terms.keys() + self.formatted_output['terms'] = list(terms.keys()) self.formatted_output['result'] = terms else: print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) @@ -191,7 +188,7 @@ def mock_hits(self, rule, start, end, index, scroll=False): if field != '_id': if not any([re.match(incl.replace('*', '.*'), field) for incl in rule['include']]): fields_to_remove.append(field) - map(doc.pop, fields_to_remove) + list(map(doc.pop, fields_to_remove)) # Separate _source and _id, convert timestamps resp = [{'_source': doc, '_id': doc['_id']} for doc in docs] @@ -211,7 +208,7 @@ def mock_terms(self, rule, start, end, index, key, qk=None, size=None): if qk is None or doc[rule['query_key']] == qk: buckets.setdefault(doc[key], 0) buckets[doc[key]] += 1 - counts = buckets.items() + counts = list(buckets.items()) counts.sort(key=lambda x: x[1], reverse=True) if size: counts = counts[:size] diff --git a/elastalert/util.py b/elastalert/util.py index c53bc3445..2fe4a8e9e 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -8,10 +8,10 @@ import dateutil.parser import pytz -from auth import Auth from six import string_types from . import ElasticSearchClient +from .auth import Auth logging.basicConfig() elastalert_logger = logging.getLogger('elastalert') @@ -27,7 +27,7 @@ def get_module(module_name): base_module = __import__(module_path, globals(), locals(), [module_class]) module = getattr(base_module, module_class) except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] + raise EAException("Could not import module %s: %s" % (module_name, e)).with_traceback(sys.exc_info()[2]) return module @@ -411,7 +411,7 @@ def parse_deadline(value): def flatten_dict(dct, delim='.', prefix=''): ret = {} - for key, val in dct.items(): + for key, val in list(dct.items()): if type(val) == dict: ret.update(flatten_dict(val, prefix=prefix + key + delim)) else: @@ -442,8 +442,8 @@ def resolve_string(string, match, missing_text=''): string = string.format(**dd_match) break except KeyError as e: - if '{%s}' % e.message not in string: + if '{%s}' % str(e).strip("'") not in string: break - string = string.replace('{%s}' % e.message, '{_missing_value}') + string = string.replace('{%s}' % str(e).strip("'"), '{_missing_value}') return string diff --git a/setup.py b/setup.py index 930b6f6cc..273f01da3 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0b2', + version='0.2.0b3', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 51b7e0f16..549b60624 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +import base64 import datetime import json import subprocess -from contextlib import nested import mock import pytest @@ -35,7 +35,7 @@ def get_match_str(self, event): def test_basic_match_string(ea): ea.rules[0]['top_count_keys'] = ['username'] match = {'@timestamp': '1918-01-17', 'field': 'value', 'top_events_username': {'bob': 10, 'mallory': 5}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'anytest' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -44,32 +44,32 @@ def test_basic_match_string(ea): # Non serializable objects don't cause errors match['non-serializable'] = {open: 10} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) # unicode objects dont cause errors - match['snowman'] = u'☃' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + match['snowman'] = '☃' + alert_text = str(BasicMatchString(ea.rules[0], match)) # Pretty printed objects match.pop('non-serializable') match['object'] = {'this': {'that': [1, 2, "3"]}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) - assert '"this": {\n "that": [\n 1, \n 2, \n "3"\n ]\n }' in alert_text + alert_text = str(BasicMatchString(ea.rules[0], match)) + assert '"this": {\n "that": [\n 1,\n 2,\n "3"\n ]\n }' in alert_text ea.rules[0]['alert_text'] = 'custom text' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'anytest' not in alert_text ea.rules[0]['alert_text_type'] = 'alert_text_only' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' not in alert_text assert 'username' not in alert_text assert 'field: value' not in alert_text ea.rules[0]['alert_text_type'] = 'exclude_fields' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -83,8 +83,8 @@ def test_jira_formatted_match_string(ea): expected_alert_text_snippet = '{code}{\n' \ + tab + '"foo": {\n' \ + 2 * tab + '"bar": [\n' \ - + 3 * tab + '"one", \n' \ - + 3 * tab + '2, \n' \ + + 3 * tab + '"one",\n' \ + + 3 * tab + '2,\n' \ + 3 * tab + '"three"\n' \ + 2 * tab + ']\n' \ + tab + '}\n' \ @@ -95,7 +95,7 @@ def test_jira_formatted_match_string(ea): def test_email(): rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -158,9 +158,9 @@ def test_email_from_field(): def test_email_with_unicode_strings(): - rule = {'name': 'test alert', 'email': u'testing@test.test', 'from_addr': 'testfrom@test.test', + rule = {'name': 'test alert', 'email': 'testing@test.test', 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -170,7 +170,7 @@ def test_email_with_unicode_strings(): mock.call().ehlo(), mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), - mock.call().sendmail(mock.ANY, [u'testing@test.test'], mock.ANY), + mock.call().sendmail(mock.ANY, ['testing@test.test'], mock.ANY), mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -329,7 +329,7 @@ def test_email_with_args(): mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) - alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': u'☃'}}]) + alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': '☃'}}]) expected = [mock.call('localhost'), mock.call().ehlo(), mock.call().has_extn('STARTTLS'), @@ -340,7 +340,7 @@ def test_email_with_args(): body = mock_smtp.mock_calls[4][1][2] # Extract the MIME encoded message body - body_text = body.split('\n\n')[-1][:-1].decode('base64') + body_text = base64.b64decode(body.split('\n\n')[-1][:-1]).decode('utf-8') assert 'testing' in body_text assert '' in body_text @@ -380,9 +380,9 @@ def test_opsgenie_basic(): alert = OpsGenieAlerter(rule) alert.alert([{'@timestamp': '2014-10-31T00:00:00'}]) - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -406,9 +406,9 @@ def test_opsgenie_frequency(): assert alert.get_info()['recipients'] == rule['opsgenie_recipients'] - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -478,10 +478,8 @@ def test_jira(): mock_priority = mock.Mock(id='5') - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = [] @@ -511,10 +509,8 @@ def test_jira(): # Search called if jira_bump_tickets rule['jira_bump_tickets'] = True - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -529,10 +525,8 @@ def test_jira(): # Remove a field if jira_ignore_in_title set rule['jira_ignore_in_title'] = 'test_term' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -545,10 +539,8 @@ def test_jira(): assert 'test_value' not in mock_jira.mock_calls[3][1][0] # Issue is still created if search_issues throws an exception - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.side_effect = JIRAError @@ -566,10 +558,8 @@ def test_jira(): # Check ticket is bumped if it is updated 4 days ago mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4)) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -584,10 +574,8 @@ def test_jira(): # Check ticket is bumped is not bumped if ticket is updated right now mock_issue.fields.updated = str(ts_now()) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -621,10 +609,8 @@ def test_jira(): mock_fields = [ {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}} ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -696,10 +682,8 @@ def test_jira_arbitrary_field_support(): }, ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -739,10 +723,8 @@ def test_jira_arbitrary_field_support(): # Reference an arbitrary string field that is not defined on the JIRA server rule['jira_nonexistent_field'] = 'nonexistent field value' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -757,10 +739,8 @@ def test_jira_arbitrary_field_support(): # Reference a watcher that does not exist rule['jira_watchers'] = 'invalid_watcher' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -1567,7 +1547,7 @@ def test_alert_text_kw(ea): 'field': 'field', } match = {'@timestamp': '1918-01-17', 'field': 'value'} - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) body = '{field} at {@timestamp}'.format(**match) assert body in alert_text @@ -1586,7 +1566,7 @@ def test_alert_text_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Priority: priority from rule' in alert_text assert 'Owner: the owner from rule' in alert_text @@ -1612,7 +1592,7 @@ def test_alert_text_kw_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Owner: the owner from rule' in alert_text assert 'Foo: foo from rule' in alert_text @@ -1990,7 +1970,7 @@ def test_alerta_no_auth(ea): 'name': 'Test Alerta rule!', 'alerta_api_url': 'http://elastalerthost:8080/api/alert', 'timeframe': datetime.timedelta(hours=1), - 'timestamp_field': u'@timestamp', + 'timestamp_field': '@timestamp', 'alerta_api_skip_ssl': True, 'alerta_attributes_keys': ["hostname", "TimestampEvent", "senderIP"], 'alerta_attributes_values': ["%(key)s", "%(logdate)s", "%(sender_ip)s"], @@ -2007,7 +1987,7 @@ def test_alerta_no_auth(ea): } match = { - u'@timestamp': '2014-10-10T00:00:00', + '@timestamp': '2014-10-10T00:00:00', # 'key': ---- missing field on purpose, to verify that simply the text is left empty # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty 'sender_ip': '1.1.1.1', diff --git a/tests/base_test.py b/tests/base_test.py index 4c6485c39..9b25f6676 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import contextlib import copy import datetime import json @@ -40,7 +39,7 @@ def generate_hits(timestamps, **kwargs): '_source': {'@timestamp': ts}, '_type': 'logs', '_index': 'idx'} - for key, item in kwargs.iteritems(): + for key, item in kwargs.items(): data['_source'][key] = item # emulate process_hits(), add metadata to _source for field in ['_id', '_type', '_index']: @@ -70,7 +69,7 @@ def test_init_rule(ea): # Simulate state of a rule just loaded from a file ea.rules[0]['minimum_starttime'] = datetime.datetime.now() new_rule = copy.copy(ea.rules[0]) - map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']) + list(map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime'])) # Properties are copied from ea.rules[0] ea.rules[0]['starttime'] = '2014-01-02T00:11:22' @@ -262,8 +261,8 @@ def test_run_rule_calls_garbage_collect(ea): end_time = '2014-09-26T12:00:00Z' ea.buffer_time = datetime.timedelta(hours=1) ea.run_every = datetime.timedelta(hours=1) - with contextlib.nested(mock.patch.object(ea.rules[0]['type'], 'garbage_collect'), - mock.patch.object(ea, 'run_query')) as (mock_gc, mock_get_hits): + with mock.patch.object(ea.rules[0]['type'], 'garbage_collect') as mock_gc, \ + mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], ts_to_dt(end_time), ts_to_dt(start_time)) # Running ElastAlert every hour for 12 hours, we should see self.garbage_collect called 12 times. @@ -624,12 +623,12 @@ def test_silence(ea): def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] - hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'☃', those=4) + hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that='☃', those=4) ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] - assert call_args[0][0][0]['this,that,those'] == u'abc, ☃, 4' + assert call_args[0][0][0]['this,that,those'] == 'abc, ☃, 4' def test_silence_query_key(ea): @@ -754,7 +753,8 @@ def test_realert_with_nested_query_key(ea): def test_count(ea): ea.rules[0]['use_count_query'] = True ea.rules[0]['doc_type'] = 'doctype' - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count') as mock_hits: ea.run_rule(ea.rules[0], END, START) # Assert that es.count is run against every run_every timeframe between START and END @@ -766,8 +766,8 @@ def test_count(ea): end = start + ea.run_every query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) + mock_hits.assert_any_call(mock.ANY, start, end, mock.ANY) start = start + ea.run_every - ea.thread_data.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -910,7 +910,8 @@ def test_set_starttime(ea): assert ea.rules[0]['starttime'] == end - ea.run_every # Count query, with previous endtime - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count'): ea.run_rule(ea.rules[0], END, START) ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == END @@ -977,7 +978,7 @@ def test_kibana_dashboard(ea): url = ea.use_kibana_link(ea.rules[0], match) db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard']) found_filters = 0 - for filter_id, filter_dict in db['services']['filter']['list'].items(): + for filter_id, filter_dict in list(db['services']['filter']['list'].items()): if (filter_dict['field'] == 'foo' and filter_dict['query'] == '"cat"') or \ (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): found_filters += 1 @@ -1113,7 +1114,7 @@ def test_exponential_realert(ea): for args in test_values: ea.silence_cache[ea.rules[0]['name']] = (args[1], args[2]) next_alert, exponent = ea.next_alert_time(ea.rules[0], ea.rules[0]['name'], args[0]) - assert exponent == next_res.next() + assert exponent == next(next_res) def test_wait_until_responsive(ea): diff --git a/tests/conftest.py b/tests/conftest.py index d3f6401ff..457eefee3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -248,7 +248,7 @@ def ea_sixsix(): def environ(): """py.test fixture to get a fresh mutable environment.""" old_env = os.environ - new_env = dict(old_env.items()) + new_env = dict(list(old_env.items())) os.environ = new_env yield os.environ os.environ = old_env diff --git a/tests/create_index_test.py b/tests/create_index_test.py index ba306aee5..47a6247dc 100644 --- a/tests/create_index_test.py +++ b/tests/create_index_test.py @@ -18,36 +18,36 @@ def test_read_default_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_5_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5) assert es_mapping in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_6_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) def test_read_default_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings() assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_5_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(5) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_6_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(6) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index 6b79eb920..997db7e1c 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -34,9 +34,9 @@ class TestElasticsearch(object): def test_create_indices(self, es_client): elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index) indices_mappings = es_client.indices.get_mapping(test_index + '*') - print('-' * 50) - print(json.dumps(indices_mappings, indent=2)) - print('-' * 50) + print(('-' * 50)) + print((json.dumps(indices_mappings, indent=2))) + print(('-' * 50)) if es_client.is_atleastsix(): assert test_index in indices_mappings assert test_index + '_error' in indices_mappings diff --git a/tests/loaders_test.py b/tests/loaders_test.py index ee1f2407e..509e8d4cb 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -53,7 +53,7 @@ def test_import_rules(): mock_open.return_value = test_rule_copy # Test that type is imported - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' @@ -63,7 +63,7 @@ def test_import_rules(): test_rule_copy = copy.deepcopy(test_rule) mock_open.return_value = test_rule_copy test_rule_copy['alert'] = 'testing2.test2.Alerter' - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.alerts rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' @@ -212,8 +212,8 @@ def test_load_rules(): with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) @@ -238,8 +238,8 @@ def test_load_default_host_port(): with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) @@ -325,7 +325,7 @@ def test_load_disabled_rules(): def test_raises_on_missing_config(): optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): + for key in list(test_rule_copy.keys()): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) test_rule_copy.pop(key) @@ -338,12 +338,11 @@ def test_raises_on_missing_config(): mock_conf_open.return_value = test_config_copy with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [('', [], ['testrule.yaml'])] with pytest.raises(EAException, message='key %s should be required' % key): rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) - print(rules) def test_compound_query_key(): diff --git a/tox.ini b/tox.ini index b8c80496a..6a6efc293 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py27,docs +envlist = py36,docs [testenv] deps = -rrequirements-dev.txt From ef1ed3433f3b8db204db250d7984e1ca6f98e102 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 10 Jun 2019 13:08:49 -0700 Subject: [PATCH 30/74] Changed travis config to py36 --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7c8ba2b34..569bf12d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: -- '2.7' +- '3.6' env: - TOXENV=docs -- TOXENV=py27 +- TOXENV=py36 install: - pip install tox - > @@ -19,16 +19,16 @@ script: make test-elasticsearch else make test - fi + fi jobs: include: - stage: 'Elasticsearch test' - env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64 - - env: TOXENV=py27 ES_VERSION=6.6.2 - - env: TOXENV=py27 ES_VERSION=6.3.2 - - env: TOXENV=py27 ES_VERSION=6.2.4 - - env: TOXENV=py27 ES_VERSION=6.0.1 - - env: TOXENV=py27 ES_VERSION=5.6.16 + env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 + - env: TOXENV=py36 ES_VERSION=6.6.2 + - env: TOXENV=py36 ES_VERSION=6.3.2 + - env: TOXENV=py36 ES_VERSION=6.2.4 + - env: TOXENV=py36 ES_VERSION=6.0.1 + - env: TOXENV=py36 ES_VERSION=5.6.16 deploy: provider: pypi From df4ef0c06e8966abd8dee3e52af02d695103449c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 15 Jul 2019 14:58:25 -0700 Subject: [PATCH 31/74] Fixed elasticsearch tests --- tests/conftest.py | 4 ++-- tests/elasticsearch_test.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 457eefee3..2b547ba41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -184,9 +184,9 @@ def ea(): ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() - ea.writeback_es.search.return_value = {'hits': {'hits': []}} + ea.writeback_es.search.return_value = {'hits': {'hits': []}, 'total': 0} ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} - ea.writeback_es.index.return_value = {'_id': 'ABCD'} + ea.writeback_es.index.return_value = {'_id': 'ABCD', 'created': True} ea.current_es = mock_es_client('', '') ea.thread_data.current_es = ea.current_es ea.thread_data.num_hits = 0 diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index 997db7e1c..308356c25 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -51,7 +51,7 @@ def test_create_indices(self, es_client): assert 'past_elastalert' in indices_mappings[test_index]['mappings'] @pytest.mark.usefixtures("ea") - def test_aggregated_alert(self, ea): # noqa: F811 + def test_aggregated_alert(self, ea, es_client): # noqa: F811 match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) ea.rules[0]['aggregate_by_match_time'] = True @@ -59,6 +59,7 @@ def test_aggregated_alert(self, ea): # noqa: F811 'num_hits': 0, 'num_matches': 3 } + ea.writeback_es = es_client res = ea.add_aggregated_alert(match, ea.rules[0]) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -70,9 +71,10 @@ def test_aggregated_alert(self, ea): # noqa: F811 assert ea.find_pending_aggregate_alert(ea.rules[0]) @pytest.mark.usefixtures("ea") - def test_silenced(self, ea): # noqa: F811 + def test_silenced(self, ea, es_client): # noqa: F811 until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) + ea.writeback_es = es_client res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -94,5 +96,7 @@ def test_get_hits(self, ea, es_client): # noqa: F811 ea.rules[0]['five'] = True else: ea.rules[0]['five'] = False + ea.thread_data.current_es = ea.current_es hits = ea.get_hits(ea.rules[0], start, end, test_index) + assert isinstance(hits, list) From ecf40ad31530c34c522d2f8162f6df8f4b32fb1c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 15 Jul 2019 16:58:23 -0700 Subject: [PATCH 32/74] Fixed tests by setting some default args --- elastalert/test_rule.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 72032de1b..629ce5a06 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -402,6 +402,13 @@ def run_rule_test(self): overwrites = { 'rules_loader': 'file', } + + # Set arguments that ElastAlerter needs + args.verbose = args.alert + args.debug = not args.verbose + args.es_debug = False + args.es_debug_trace = False + conf = load_conf(args, defaults, overwrites) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From aaeec1bb3acaf256f8e025a21d9ec408ca03cc6a Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Tue, 23 Jul 2019 16:27:01 -0700 Subject: [PATCH 33/74] Remove dead links. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8757290f6..0557c2744 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` -[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) -[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From b65b9718075677b313e42561eab2eb524534ea01 Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Wed, 24 Jul 2019 18:10:32 +0200 Subject: [PATCH 34/74] Adds to the elasticsearch mapping a dynaic_templates configuration. Now, aggregations can be performed on alerts, for example in visualisations. This also enables ElastAlert to run over ElastAlert alerts. --- elastalert/es_mappings/6/elastalert.json | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index ecc30c8ed..645a67762 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -1,4 +1,17 @@ { + "numeric_detection": true, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "rule_name": { "type": "keyword" @@ -16,8 +29,7 @@ "format": "dateOptionalTime" }, "match_body": { - "type": "object", - "enabled": "false" + "type": "object" }, "aggregate_id": { "type": "keyword" From 4b927b95e263a21b8654a8d18a2a4a3a8a37d551 Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Thu, 25 Jul 2019 13:09:28 +0200 Subject: [PATCH 35/74] Enables querying the source index and correlate with ElastAlert. Add support to put the match in the root ElastAlert document. For example, if your query_key is source.ip, source.ip will also be present in the ElastAlert index. --- elastalert/elastalert.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 11e037a6e..b68fa671b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1460,6 +1460,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No 'alert_time': alert_time } + if rule.get('include_match_in_root'): + body.update({k: v for k, v in match.items() if not k.startswith('_')}) + if self.add_metadata_alert: body['category'] = rule['category'] body['description'] = rule['description'] From bb248f9b550fa8b3ef8d999e0b1f00cf36bed7dd Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Thu, 25 Jul 2019 13:10:23 +0200 Subject: [PATCH 36/74] Add sample rules SSH usees include_match_in_root to give an example of the feature. Repeat offender is an example of a second order rule. --- example_rules/ssh-repeat-offender.yaml | 61 ++++++++++++++++++++++++ example_rules/ssh.yaml | 64 ++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 example_rules/ssh-repeat-offender.yaml create mode 100644 example_rules/ssh.yaml diff --git a/example_rules/ssh-repeat-offender.yaml b/example_rules/ssh-repeat-offender.yaml new file mode 100644 index 000000000..27a439fcd --- /dev/null +++ b/example_rules/ssh-repeat-offender.yaml @@ -0,0 +1,61 @@ +# Rule name, must be unique +name: SSH abuse - reapeat offender + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 2 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + weeks: 1 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + - term: + rule_name: "SSH abuse" + +index: elastalert + +# When the attacker continues, send a new alert after x minutes +realert: + weeks: 4 + +query_key: + - match_body.source.ip + +include: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +alert_subject: "SSH abuse (repeat offender) on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - match_body.host.hostname + - kibana_link + +alert_text: |- + An reapeat offender has been active on {}. + + IP: {} + User: {} +alert_text_args: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +# The alert is use when a match is found +alert: + - slack + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml new file mode 100644 index 000000000..7af890784 --- /dev/null +++ b/example_rules/ssh.yaml @@ -0,0 +1,64 @@ +# Rule name, must be unique + name: SSH abuse (ElastAlert 3.0.1) - 2 + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 20 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + minutes: 60 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: +- query: + query_string: + query: "event.type:authentication_failure" + +index: auditbeat-* + +# When the attacker continues, send a new alert after x minutes +realert: + minutes: 1 + +query_key: + - source.ip + +include: + - host.hostname + - user.name + - source.ip + +include_match_in_root: true + +alert_subject: "SSH abuse on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - host.hostname + - kibana_link + +alert_text: |- + An attack on {} is detected. + The attacker looks like: + User: {} + IP: {} +alert_text_args: + - host.hostname + - user.name + - source.ip + +# The alert is use when a match is found +alert: + - debug + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" From 70dab8f49b6ea259d2fd26359d6f0533d394e902 Mon Sep 17 00:00:00 2001 From: Luke Watson Date: Mon, 29 Jul 2019 11:06:50 -0400 Subject: [PATCH 37/74] Add "http_post_headers" to documentation --- docs/source/ruletypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f8452dcdd..ba43a187f 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2037,6 +2037,8 @@ Optional: ``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here. +``http_post_headers``: Key:value pairs of headers to be sent as part of the request. + ``http_post_proxy``: URL of proxy, if required. ``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False. @@ -2051,6 +2053,8 @@ Example usage:: ip: clientip http_post_static_payload: apikey: abc123 + http_post_headers: + authorization: Basic 123dr3234 Alerter From f6ce4223400cb1f880d5c390515c77deb4b232b2 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 11:26:41 -0700 Subject: [PATCH 38/74] Fixed deprecated search --- elastalert/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 05d9ef7ff..d0d850aa0 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -245,4 +245,4 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): index = "_all" return self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body - ) + )[1] From 92d7327668d84bf6dc1143f01af3c4b5afd9a338 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 11:30:24 -0700 Subject: [PATCH 39/74] Remove doc_type warning --- elastalert/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index d0d850aa0..30e3ead9a 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import copy -import logging from elasticsearch import Elasticsearch from elasticsearch import RequestsHttpConnection @@ -235,8 +234,6 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): :arg version: Specify whether to return document version as part of a hit """ - logging.warning( - 'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8') # from is a reserved word so it cannot be used, use from_ instead if "from_" in params: params["from"] = params.pop("from_") From 6950ea80104c190703eaa8fa19b4add558ce6717 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 12:07:55 -0700 Subject: [PATCH 40/74] Fix deprecated_search return type for all versions --- elastalert/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 30e3ead9a..daad8eb29 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -240,6 +240,9 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): if not index: index = "_all" - return self.transport.perform_request( + res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body - )[1] + ) + if type(res) == list: + return res[1] + return res From 007f4a50cc18052bc8998a18d7950525833aa673 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 16:04:59 -0700 Subject: [PATCH 41/74] Add retries and caching to ES info --- elastalert/__init__.py | 9 ++++++++- elastalert/elastalert.py | 5 ++++- tests/base_test.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index daad8eb29..86b2a3735 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import copy +import time from elasticsearch import Elasticsearch from elasticsearch import RequestsHttpConnection from elasticsearch.client import _make_path from elasticsearch.client import query_params +from elasticsearch.exceptions import TransportError class ElasticSearchClient(Elasticsearch): @@ -42,7 +44,12 @@ def es_version(self): Returns the reported version from the Elasticsearch server. """ if self._es_version is None: - self._es_version = self.info()['version']['number'] + for retry in range(3): + try: + self._es_version = self.info()['version']['number'] + break + except TransportError: + time.sleep(3) return self._es_version def is_atleastfive(self): diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 98e9dc8f0..4ac9e1e56 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -105,6 +105,7 @@ def parse_args(self, args): self.args = parser.parse_args(args) def __init__(self, args): + self.es_clients = {} self.parse_args(args) self.debug = self.args.debug self.verbose = self.args.verbose @@ -843,7 +844,7 @@ def run_rule(self, rule, endtime, starttime=None): :return: The number of matches that the rule produced. """ run_start = time.time() - self.thread_data.current_es = elasticsearch_client(rule) + self.thread_data.current_es = self.es_clients.setdefault(rule['name'], elasticsearch_client(rule)) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -1115,6 +1116,8 @@ def load_rule_changes(self): continue if self.init_rule(new_rule): elastalert_logger.info('Loaded new rule %s' % (rule_file)) + if new_rule['name'] in self.es_clients: + self.es_clients.pop(new_rule['name']) self.rules.append(new_rule) self.rule_hashes = new_rule_hashes diff --git a/tests/base_test.py b/tests/base_test.py index 9b25f6676..15474c690 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -516,7 +516,7 @@ def test_agg_no_writeback_connectivity(ea): ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], END, START) ea.add_aggregated_alert.assert_any_call({'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3}, ea.rules[0]) From e79eb4041a6732c190dfe3d1bf4a19d2388cc15c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 17:17:31 -0700 Subject: [PATCH 42/74] Reraise error if retries fail (get_info) --- elastalert/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 86b2a3735..f2b868c81 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -49,6 +49,8 @@ def es_version(self): self._es_version = self.info()['version']['number'] break except TransportError: + if retry == 2: + raise time.sleep(3) return self._es_version From 3d103eef83e3d25f9dc353c36825a98becc380a3 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 1 Aug 2019 16:23:05 -0700 Subject: [PATCH 43/74] Update elastalert/test_rule.py Co-Authored-By: Reid Miller --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 629ce5a06..627dfe82c 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -405,7 +405,7 @@ def run_rule_test(self): # Set arguments that ElastAlerter needs args.verbose = args.alert - args.debug = not args.verbose + args.debug = not args.alert args.es_debug = False args.es_debug_trace = False From 3555faee35d95ec847a781b844e10ef5d5c68d43 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 2 Aug 2019 13:37:25 -0700 Subject: [PATCH 44/74] Fix deprecated_Search --- elastalert/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index f2b868c81..55bfdb32f 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -252,6 +252,6 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body ) - if type(res) == list: + if type(res) == list or type(res) == tuple: return res[1] return res From 15ed5772331765c529b7c5475d5086f296792aff Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 7 Aug 2019 13:47:38 -0700 Subject: [PATCH 45/74] Version 0.2.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 273f01da3..c71af3d30 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0b3', + version='0.2.0', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', setup_requires='setuptools', license='Copyright 2014 Yelp', classifiers=[ - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], From f30de3915adf10c9112c54651f69bcbf5579c6ab Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 7 Aug 2019 13:59:26 -0700 Subject: [PATCH 46/74] Remove ES7 note --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 0557c2744..9ce2ad445 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` - - [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From 2ba314ecb3ac236493e512064105fd247e70a382 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 8 Aug 2019 14:49:54 -0700 Subject: [PATCH 47/74] Fix a bug introduced in 2.0 merge --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 30f32c03e..cafb6ef57 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -657,7 +657,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if 'scroll_id' in rule: scroll_id = rule.pop('scroll_id') - self.current_es.clear_scroll(scroll_id=scroll_id) + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) return True From 4dd50dc3f1147d692b66ca05c3ec8642a70e9af5 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 8 Aug 2019 14:56:49 -0700 Subject: [PATCH 48/74] Version 0.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c71af3d30..412d53b1e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0', + version='0.2.1', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From a5322f25b80828cc2183970c9c9f81f32867ce3b Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Mon, 12 Aug 2019 20:33:35 +0530 Subject: [PATCH 49/74] Catch Scroll Clear NotFound Error --- elastalert/elastalert.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index cafb6ef57..5436ada18 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -24,6 +24,7 @@ from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException from elasticsearch.exceptions import TransportError +from elasticsearch.exceptions import NotFoundError from . import kibana from .alerts import DebugAlerter @@ -657,7 +658,10 @@ def run_query(self, rule, start=None, end=None, scroll=False): if 'scroll_id' in rule: scroll_id = rule.pop('scroll_id') - self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + try: + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + except NotFoundError: + pass return True From 1df5e8feff119ded9b975e8cfab71820c0628f6e Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 12 Aug 2019 12:41:16 -0700 Subject: [PATCH 50/74] update requirements, docs, changelog --- README.md | 2 ++ changelog.md | 17 +++++++++++++++++ docs/source/running_elastalert.rst | 2 +- requirements.txt | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9ce2ad445..94a091aef 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. + [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) diff --git a/changelog.md b/changelog.md index 30b52f601..fe30b573c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,22 @@ # Change Log +# v0.2.1 + +### Fixed +- Fixed an AttributeError introduced in 0.2.0 + +# v0.2.0 + +- Switched to Python 3 + +### Added +- Add rule loader class for customized rule loading +- Added thread based rules and limit_execution +- Run_every can now be customized per rule + +### Fixed +- Various small fixes + # v0.1.39 ### Added diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 09e307c24..7fdf1eeba 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -8,7 +8,7 @@ Requirements - Elasticsearch - ISO8601 or Unix timestamped data -- Python 2.7 +- Python 3.6 - pip, see requirements.txt - Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev diff --git a/requirements.txt b/requirements.txt index e70af5896..3e5cbed90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 python-magic>=0.4.15 python-magic>=0.4.15 -PyYAML>=3.12 +PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 From 3affdd7ebf2b6d82c5cfcad0092df97dc27cd8b1 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 13 Aug 2019 10:21:32 -0700 Subject: [PATCH 51/74] Fixed config.yaml default regression --- elastalert/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elastalert/config.py b/elastalert/config.py index c912122e0..5ae9a26e6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -41,7 +41,13 @@ def load_conf(args, defaults=None, overwrites=None): :return: The global configuration, a dictionary. """ filename = args.config - conf = yaml_loader(filename) + if filename: + conf = yaml_loader(filename) + else: + try: + conf = yaml_loader('config.yaml') + except FileNotFoundError: + raise EAException('No --config or config.yaml found') # init logging from config and set log levels according to command line options configure_logging(args, conf) From 1d3c1021c6c06b08f8483e2e0eb835bbc61ac21c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Aug 2019 09:56:11 -0400 Subject: [PATCH 52/74] Remove hive_port When using TheHive in a configuration that is not top-level (`https://IP/thehive/`), the current code does not work, as it concatenates `hive_host` & `hive_port` which creates `https://IP/thehive/:443` This tweak removes the need to explicitly specify a port and instead specify only `hive_host`. A couple examples: `hive_host: https://192.168.15.23:8080` `hive_host: https://192.168.15.23/thehive/` `hive_host: https://192.168.15.23:8080/thehive/` This tweak should be considered a breaking change, as current users will need to update their `hive_connection` configuration if `hive_url` does not contain the required port. --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3ee892d4..927ced575 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2111,7 +2111,7 @@ def alert(self, matches): connection_details = self.rule['hive_connection'] api = TheHiveApi( - '{hive_host}:{hive_port}'.format(**connection_details), + connection_details.get('hive_host'), connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) From d75a07f38eacfbef104f48be913f1f77bf79aa2b Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Fri, 23 Aug 2019 13:07:04 +0530 Subject: [PATCH 53/74] Fix Running ElastAlert instructions in README Fix #2417 , this comes after the switch to Python3, 3 months back --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94a091aef..99acc02e7 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,23 @@ In addition to this basic usage, there are many other features that make alerts To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). ## Running ElastAlert +You can either install the latest released version of ElastAlert using pip: -``$ python elastalert/elastalert.py [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` +```pip install elastalert``` + +or you can clone the ElastAlert repository for the most recent changes: + +```git clone https://github.com/Yelp/elastalert.git``` + +Install the module: + +```pip install "setuptools>=11.3"``` + +```python setup.py install``` + +The following invocation can be used to run ElastAlert after installing + +``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` ``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. From 0022a01f4cca0a83d4f26eed6ef137fcdae65f55 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 3 Sep 2019 09:14:56 -0400 Subject: [PATCH 54/74] convert str to byte before pipe it to command --- elastalert/alerts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3ee892d4..84b0ae482 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -918,10 +918,10 @@ def alert(self, matches): if self.rule.get('pipe_match_json'): match_json = json.dumps(matches, cls=DateTimeEncoder) + '\n' - stdout, stderr = subp.communicate(input=match_json) + stdout, stderr = subp.communicate(input=match_json.encode()) elif self.rule.get('pipe_alert_text'): alert_text = self.create_alert_body(matches) - stdout, stderr = subp.communicate(input=alert_text) + stdout, stderr = subp.communicate(input=alert_text.encode()) if self.rule.get("fail_on_non_zero_exit", False) and subp.wait(): raise EAException("Non-zero exit code while running command %s" % (' '.join(command))) except OSError as e: From 0a7e71bd658366d115e867b46f31209ba16ba3ca Mon Sep 17 00:00:00 2001 From: Caleb Collins-Parks <46505081+caleb15@users.noreply.github.com> Date: Fri, 6 Sep 2019 16:55:06 -0700 Subject: [PATCH 55/74] update jira to version that works with 3.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 412d53b1e..cb5e10eea 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'elasticsearch>=7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', - 'jira>=1.0.10,<1.0.15', + 'jira>=2.0.0', 'jsonschema>=2.6.0,<3.0.0', 'mock>=2.0.0', 'PyStaticConfiguration>=0.10.3', From 16a464e208301e4e8e170bd6375b5cf65231346e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 12 Sep 2019 11:34:04 -0400 Subject: [PATCH 56/74] Updating the docker test file to work with python 3.6 --- Dockerfile-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 761a777c6..3c153e644 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,9 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip +RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert ADD requirements*.txt ./ -RUN pip install -r requirements-dev.txt +RUN pip3 install -r requirements-dev.txt From cfa98a12bb7739cfc21a16a418fe95375277f1fd Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 13 Sep 2019 13:49:25 -0400 Subject: [PATCH 57/74] Switching test_rule to use load_yaml --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 627dfe82c..ee467031b 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -410,7 +410,7 @@ def run_rule_test(self): args.es_debug_trace = False conf = load_conf(args, defaults, overwrites) - rule_yaml = conf['rules_loader'].get_yaml(args.file) + rule_yaml = conf['rules_loader'].load_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: From 707b2a58a0599e1e21e9e0688314387046841250 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Wed, 18 Sep 2019 13:18:18 -0400 Subject: [PATCH 58/74] Handling non-compound query keys defined as an array --- elastalert/loaders.py | 12 +++++++++--- tests/loaders_test.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..ad05c6af2 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -324,9 +324,15 @@ def _dt_to_ts_with_format(dt): if 'include' in rule and type(rule['include']) != list: raise EAException('include option must be a list') - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) + raw_query_key = rule.get('query_key') + if isinstance(raw_query_key, list): + if len(raw_query_key) > 1: + rule['compound_query_key'] = raw_query_key + rule['query_key'] = ','.join(raw_query_key) + elif len(raw_query_key) == 1: + rule['query_key'] = raw_query_key[0] + else: + del(rule['query_key']) if isinstance(rule.get('aggregation_key'), list): rule['compound_aggregation_key'] = rule['aggregation_key'] diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 509e8d4cb..40399d969 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -358,6 +358,29 @@ def test_compound_query_key(): assert test_rule_copy['compound_query_key'] == ['field1', 'field2'] +def test_query_key_with_single_value(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = ['field1'] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'field1' in test_rule_copy['include'] + assert test_rule_copy['query_key'] == 'field1' + assert 'compound_query_key' not in test_rule_copy + + +def test_query_key_with_no_values(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = [] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'query_key' not in test_rule_copy + assert 'compound_query_key' not in test_rule_copy + + def test_name_inference(): test_config_copy = copy.deepcopy(test_config) rules_loader = FileRulesLoader(test_config_copy) From 070bb1cd0ecad1d0de66ea57de408528d5fd4347 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 19 Sep 2019 14:00:12 -0400 Subject: [PATCH 59/74] Adding support for generating Kibana Discover app link --- docs/source/ruletypes.rst | 93 ++++ elastalert/elastalert.py | 6 + elastalert/kibana_discover.py | 192 ++++++++ elastalert/loaders.py | 4 + elastalert/schema.yaml | 9 + requirements.txt | 1 + setup.py | 1 + tests/kibana_discover_test.py | 858 ++++++++++++++++++++++++++++++++++ tests/loaders_test.py | 20 + 9 files changed, 1184 insertions(+) create mode 100644 elastalert/kibana_discover.py create mode 100644 tests/kibana_discover_test.py diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ba43a187f..6e606401a 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,6 +58,20 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | +| ``use_kibana_discover`` (boolean, default False) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_url`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_version`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_index_pattern_id`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_columns`` (list of strs, default _source) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_from_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_to_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | | ``use_local_time`` (boolean, default True) | | +--------------------------------------------------------------+ | | ``realert`` (time, default: 1 min) | | @@ -510,6 +524,85 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` +use_kibana_discover +^^^^^^^^^^^^^^^^^^^ + +``use_kibana_discover``: Enables the generation of the ``kibana_link`` variable for the Kibana Discover application. +This setting requires the following settings are also configured: + +- ``kibana_discover_url`` +- ``kibana_discover_version`` +- ``kibana_discover_index_pattern_id`` + +``use_kibana_discover: true`` + +kibana_discover_url +^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_link`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. + +``kibana_discover_url: http://kibana:5601/#/discover`` + +kibana_discover_version +^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_version``: Specifies the version of the Kibana Discover application. + +The currently supported versions of Kibana Discover are: + +- `5.6` +- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` +- `7.0`, `7.1`, `7.2`, `7.3` + +``kibana_discover_version: '7.3'`` + +kibana_discover_index_pattern_id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. +These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. + +Example export of an index pattern's saved object: + +.. code-block:: text + + [ + { + "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c", + "_type": "index-pattern", + "_source": { ... } + } + ] + +You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing. + +``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` + +kibana_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. +Defaults to the ``_source`` column. + +``kibana_discover_columns: [ timestamp, message ]`` + +kibana_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``kibana_discover_from_timedelta: minutes: 2`` + +kibana_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``kibana_discover_to_timedelta: minutes: 2`` + use_local_time ^^^^^^^^^^^^^^ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5436ada18..fdbdf076e 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,6 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana +from .kibana_discover import kibana_discover_url from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1497,6 +1498,11 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link + if rule.get('use_kibana_discover'): + kb_link = kibana_discover_url(rule, matches[0]) + if kb_link: + matches[0]['kibana_link'] = kb_link + # Enhancements were already run at match time if # run_enhancements_first is set or # retried==True, which means this is a retry of a failed alert diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py new file mode 100644 index 000000000..edbc21b62 --- /dev/null +++ b/elastalert/kibana_discover.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import lookup_es_key +from .util import ts_add + +kibana_default_timedelta = datetime.timedelta(minutes=10) + +kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) + +def kibana_discover_url(rule, match): + ''' Creates a link for a kibana discover app. ''' + + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + discover_url = rule.get('kibana_discover_url') + if not discover_url: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('kibana_discover_index_pattern_id') + if not index: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('kibana_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', kibana_default_timedelta) + from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if kibana_version in kibana5_kibana6_versions: + globalState = kibana6_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + elif kibana_version in kibana7_versions: + globalState = kibana7_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + else: + logging.warning( + 'Unknown kibana discover application version %s for rule %s' % ( + kibana_version, + rule.get('name', '') + ) + ) + return None + + return "%s?_g=%s&_a=%s" % ( + os.path.expandvars(discover_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState) + ) + + +def kibana6_disover_global_state(from_time, to_time): + return prison.dumps( { + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'mode': 'absolute', + 'to': to_time + } + } ) + + +def kibana7_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def kibana_discover_app_state(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + bool_filter = { 'must': filters } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'bool': bool_filter, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'bool', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'columns': columns, + 'filters': app_filters, + 'index': index, + 'interval': 'auto' + } ) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..806e22584 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -257,6 +257,10 @@ def load_options(self, rule, conf, filename, args=None): rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) if 'kibana4_end_timedelta' in rule: rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + if 'kibana_discover_from_timedelta' in rule: + rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + if 'kibana_discover_to_timedelta' in rule: + rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) except (KeyError, TypeError) as e: raise EAException('Invalid time format used: %s' % e) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 13562c332..953bed79f 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -203,6 +203,15 @@ properties: replace_dots_in_field_names: {type: boolean} scan_entire_timeframe: {type: boolean} + ### Kibana Discover App Link + use_kibana_discover: {type: boolean} + kibana_discover_url: {type: string} + kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_index_pattern_id: {type: string} + kibana_discover_columns: {type: array, items: {type: string}} + kibana_discover_from_timedelta: *timeframe + kibana_discover_to_timedelta: *timeframe + # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..0bd9521a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ exotel>=0.1.3 jira>=1.0.10,<1.0.15 jsonschema>=2.6.0 mock>=2.0.0 +prison>=0.1.2 py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 diff --git a/setup.py b/setup.py index 412d53b1e..26445bc20 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ 'jira>=1.0.10,<1.0.15', 'jsonschema>=2.6.0,<3.0.0', 'mock>=2.0.0', + 'prison>=0.1.2', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=3.12', diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py new file mode 100644 index 000000000..cb3f97630 --- /dev/null +++ b/tests/kibana_discover_test.py @@ -0,0 +1,858 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta +import pytest + +from elastalert.kibana_discover import kibana_discover_url + + +@pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +def test_kibana_discover_url_with_kibana_7x(kibana_version): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_missing_kibana_discover_version(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_missing_kibana_discover_url(): + url = kibana_discover_url( + rule={ + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_invalid_kibana_version(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '4.5', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_discover_url_env_substitution(environ): + environ.update({ + 'KIBANA_HOST': 'kibana', + 'KIBANA_PORT': '5601', + }) + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_from_timedelta(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A10%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_to_timedelta(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A50%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T04%3A10%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A50%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_custom_columns(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'kibana_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28level%2Cmessage%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_single_filter(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_multiple_filters(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B' # value start + + '%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D' + + '%5D%7D%27' # value end + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_int_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3A200%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3A%27200%27' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # reponse start + + 'query%3A200%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_str_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_null_query_key_value(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': None + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_missing_query_key_value(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_compound_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # geo.src filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3ACA%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3ACA' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.src%3A%28' # reponse start + + 'query%3ACA%2C' + + 'type%3Aphrase' + + '%29' # geo.src end + + '%29' # match end + + '%29' # query end + + '%29%2C' # geo.src filter end + + + '%28' # geo.dest filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3AUS%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3AUS' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3AUS%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # geo.dest filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_filter_and_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29%2C' # filter end + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'status%3A%28' # status start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # status end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 509e8d4cb..38793878c 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -395,3 +395,23 @@ def test_raises_on_bad_generate_kibana_filters(): test_rule_copy['filter'] = good + bad with pytest.raises(EAException): rules_loader.load_configuration('blah', test_config) + + +def test_kibana_discover_from_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_from_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_from_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_from_timedelta'] == datetime.timedelta(minutes=2) + + +def test_kibana_discover_to_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_to_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) From 89b96ade4ace42ed37427404613372e759fadae7 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 11:27:21 -0400 Subject: [PATCH 60/74] Renaming the kibana_discover_link function --- elastalert/elastalert.py | 4 +- elastalert/kibana_discover.py | 2 +- tests/kibana_discover_test.py | 86 +++++++++++++++++------------------ 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index fdbdf076e..c006b2ae3 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,7 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana -from .kibana_discover import kibana_discover_url +from .kibana_discover import generate_kibana_discover_link from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1499,7 +1499,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): matches[0]['kibana_link'] = kb_link if rule.get('use_kibana_discover'): - kb_link = kibana_discover_url(rule, matches[0]) + kb_link = generate_kibana_discover_link(rule, matches[0]) if kb_link: matches[0]['kibana_link'] = kb_link diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index edbc21b62..1c5a27cc7 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -16,7 +16,7 @@ kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) -def kibana_discover_url(rule, match): +def generate_kibana_discover_link(rule, match): ''' Creates a link for a kibana discover app. ''' kibana_version = rule.get('kibana_discover_version') diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index cb3f97630..8e5060d80 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -2,12 +2,12 @@ from datetime import timedelta import pytest -from elastalert.kibana_discover import kibana_discover_url +from elastalert.kibana_discover import generate_kibana_discover_link @pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, @@ -39,8 +39,8 @@ def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): @pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) -def test_kibana_discover_url_with_kibana_7x(kibana_version): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, @@ -71,8 +71,8 @@ def test_kibana_discover_url_with_kibana_7x(kibana_version): assert url == expectedUrl -def test_kibana_discover_url_with_missing_kibana_discover_version(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_index_pattern_id': 'logs', @@ -86,8 +86,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_version(): assert url is None -def test_kibana_discover_url_with_missing_kibana_discover_url(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs', @@ -101,8 +101,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_url(): assert url is None -def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -116,8 +116,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): assert url is None -def test_kibana_discover_url_with_invalid_kibana_version(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_invalid_kibana_version(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '4.5', @@ -131,12 +131,12 @@ def test_kibana_discover_url_with_invalid_kibana_version(): assert url is None -def test_kibana_discover_url_with_discover_url_env_substitution(environ): +def test_generate_kibana_discover_link_with_discover_url_env_substitution(environ): environ.update({ 'KIBANA_HOST': 'kibana', 'KIBANA_PORT': '5601', }) - url = kibana_discover_url( + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', 'kibana_discover_version': '6.8', @@ -167,8 +167,8 @@ def test_kibana_discover_url_with_discover_url_env_substitution(environ): assert url == expectedUrl -def test_kibana_discover_url_with_from_timedelta(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_from_timedelta(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -200,8 +200,8 @@ def test_kibana_discover_url_with_from_timedelta(): assert url == expectedUrl -def test_kibana_discover_url_with_from_timedelta_and_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -234,8 +234,8 @@ def test_kibana_discover_url_with_from_timedelta_and_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_to_timedelta(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_to_timedelta(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -267,8 +267,8 @@ def test_kibana_discover_url_with_to_timedelta(): assert url == expectedUrl -def test_kibana_discover_url_with_to_timedelta_and_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -301,8 +301,8 @@ def test_kibana_discover_url_with_to_timedelta_and_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -334,8 +334,8 @@ def test_kibana_discover_url_with_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_custom_columns(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_custom_columns(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -367,8 +367,8 @@ def test_kibana_discover_url_with_custom_columns(): assert url == expectedUrl -def test_kibana_discover_url_with_single_filter(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_single_filter(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -418,8 +418,8 @@ def test_kibana_discover_url_with_single_filter(): assert url == expectedUrl -def test_kibana_discover_url_with_multiple_filters(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_multiple_filters(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -472,8 +472,8 @@ def test_kibana_discover_url_with_multiple_filters(): assert url == expectedUrl -def test_kibana_discover_url_with_int_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_int_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -532,8 +532,8 @@ def test_kibana_discover_url_with_int_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_str_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_str_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -594,8 +594,8 @@ def test_kibana_discover_url_with_str_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_null_query_key_value(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_null_query_key_value(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -644,8 +644,8 @@ def test_kibana_discover_url_with_null_query_key_value(): assert url == expectedUrl -def test_kibana_discover_url_with_missing_query_key_value(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_query_key_value(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -693,8 +693,8 @@ def test_kibana_discover_url_with_missing_query_key_value(): assert url == expectedUrl -def test_kibana_discover_url_with_compound_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_compound_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -781,8 +781,8 @@ def test_kibana_discover_url_with_compound_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_filter_and_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_filter_and_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', From 47cad175dce5966edadbe3abd553b667325d2543 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 11:35:08 -0400 Subject: [PATCH 61/74] Pruning the duplicate requirements --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..d5fbaa46a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 cffi>=1.11.5 -cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 elasticsearch>=7.0.0 @@ -16,7 +15,6 @@ py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 python-magic>=0.4.15 -python-magic>=0.4.15 PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 From e5d20eaa0c79e9053c320a3211ed41f9271855b5 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 12:45:59 -0400 Subject: [PATCH 62/74] Generate kibana discover link (#3) * Renaming config to generate-kibana-discover-link --- docs/source/ruletypes.rst | 12 ++++++------ elastalert/elastalert.py | 4 ++-- elastalert/kibana_discover.py | 14 +++++++------- elastalert/schema.yaml | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 6e606401a..d550a8c93 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,7 +58,7 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | -| ``use_kibana_discover`` (boolean, default False) | | +| ``generate_kibana_discover_link`` (boolean, default False) | | +--------------------------------------------------------------+ | | ``kibana_discover_url`` (string, no default) | | +--------------------------------------------------------------+ | @@ -524,22 +524,22 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` -use_kibana_discover -^^^^^^^^^^^^^^^^^^^ +generate_kibana_discover_link +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``use_kibana_discover``: Enables the generation of the ``kibana_link`` variable for the Kibana Discover application. +``generate_kibana_discover_link``: Enables the generation of the ``kibana_discover_link`` variable for the Kibana Discover application. This setting requires the following settings are also configured: - ``kibana_discover_url`` - ``kibana_discover_version`` - ``kibana_discover_index_pattern_id`` -``use_kibana_discover: true`` +``generate_kibana_discover_link: true`` kibana_discover_url ^^^^^^^^^^^^^^^^^^^^ -``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_link`` variable. +``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_link`` variable. This value can use `$VAR` and `${VAR}` references to expand environment variables. ``kibana_discover_url: http://kibana:5601/#/discover`` diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index c006b2ae3..5cc82b554 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1498,10 +1498,10 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link - if rule.get('use_kibana_discover'): + if rule.get('generate_kibana_discover_link'): kb_link = generate_kibana_discover_link(rule, matches[0]) if kb_link: - matches[0]['kibana_link'] = kb_link + matches[0]['kibana_discover_link'] = kb_link # Enhancements were already run at match time if # run_enhancements_first is set or diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 1c5a27cc7..729e0ea29 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -19,19 +19,19 @@ def generate_kibana_discover_link(rule, match): ''' Creates a link for a kibana discover app. ''' - kibana_version = rule.get('kibana_discover_version') - if not kibana_version: + discover_url = rule.get('kibana_discover_url') + if not discover_url: logging.warning( - 'use_kibana_discover was configured without kibana_discover_version for rule %s' % ( + 'Missing kibana_discover_url for rule %s' % ( rule.get('name', '') ) ) return None - discover_url = rule.get('kibana_discover_url') - if not discover_url: + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: logging.warning( - 'use_kibana_discover was configured without kibana_discover_url for rule %s' % ( + 'Missing kibana_discover_version for rule %s' % ( rule.get('name', '') ) ) @@ -40,7 +40,7 @@ def generate_kibana_discover_link(rule, match): index = rule.get('kibana_discover_index_pattern_id') if not index: logging.warning( - 'use_kibana_discover was configured without kibana_discover_index_pattern_id for rule %s' % ( + 'Missing kibana_discover_index_pattern_id for rule %s' % ( rule.get('name', '') ) ) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 953bed79f..8a7294a6f 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -204,7 +204,7 @@ properties: scan_entire_timeframe: {type: boolean} ### Kibana Discover App Link - use_kibana_discover: {type: boolean} + generate_kibana_discover_link: {type: boolean} kibana_discover_url: {type: string} kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} From 82760140b83672f6dced41bf28053fdcb1fa31cf Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 13:53:59 -0400 Subject: [PATCH 63/74] Renaming to generate_kibana_discover_url (#4) --- docs/source/ruletypes.rst | 32 ++++----- elastalert/elastalert.py | 8 +-- elastalert/kibana_discover.py | 10 +-- elastalert/schema.yaml | 4 +- tests/kibana_discover_test.py | 126 +++++++++++++++++----------------- 5 files changed, 90 insertions(+), 90 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d550a8c93..9b827b753 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,9 +58,9 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | -| ``generate_kibana_discover_link`` (boolean, default False) | | +| ``generate_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | -| ``kibana_discover_url`` (string, no default) | | +| ``kibana_discover_app_url`` (string, no default) | | +--------------------------------------------------------------+ | | ``kibana_discover_version`` (string, no default) | | +--------------------------------------------------------------+ | @@ -524,28 +524,28 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` -generate_kibana_discover_link -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +generate_kibana_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``generate_kibana_discover_link``: Enables the generation of the ``kibana_discover_link`` variable for the Kibana Discover application. +``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application. This setting requires the following settings are also configured: -- ``kibana_discover_url`` +- ``kibana_discover_app_url`` - ``kibana_discover_version`` - ``kibana_discover_index_pattern_id`` -``generate_kibana_discover_link: true`` +``generate_kibana_discover_url: true`` -kibana_discover_url -^^^^^^^^^^^^^^^^^^^^ +kibana_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^ -``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_link`` variable. +``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable. This value can use `$VAR` and `${VAR}` references to expand environment variables. -``kibana_discover_url: http://kibana:5601/#/discover`` +``kibana_discover_app_url: http://kibana:5601/#/discover`` kibana_discover_version -^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_version``: Specifies the version of the Kibana Discover application. @@ -558,7 +558,7 @@ The currently supported versions of Kibana Discover are: ``kibana_discover_version: '7.3'`` kibana_discover_index_pattern_id -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. @@ -580,7 +580,7 @@ You can modify an index pattern's id by exporting the saved object, modifying th ``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` kibana_discover_columns -^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. Defaults to the ``_source`` column. @@ -588,7 +588,7 @@ Defaults to the ``_source`` column. ``kibana_discover_columns: [ timestamp, message ]`` kibana_discover_from_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. @@ -596,7 +596,7 @@ The `from` time is calculated by subtracting this timedelta from the event time. ``kibana_discover_from_timedelta: minutes: 2`` kibana_discover_to_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5cc82b554..01beb57d8 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,7 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana -from .kibana_discover import generate_kibana_discover_link +from .kibana_discover import generate_kibana_discover_url from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1498,10 +1498,10 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link - if rule.get('generate_kibana_discover_link'): - kb_link = generate_kibana_discover_link(rule, matches[0]) + if rule.get('generate_kibana_discover_url'): + kb_link = generate_kibana_discover_url(rule, matches[0]) if kb_link: - matches[0]['kibana_discover_link'] = kb_link + matches[0]['kibana_discover_url'] = kb_link # Enhancements were already run at match time if # run_enhancements_first is set or diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 729e0ea29..7e4dbb5d1 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -16,13 +16,13 @@ kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) -def generate_kibana_discover_link(rule, match): +def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' - discover_url = rule.get('kibana_discover_url') - if not discover_url: + discover_app_url = rule.get('kibana_discover_app_url') + if not discover_app_url: logging.warning( - 'Missing kibana_discover_url for rule %s' % ( + 'Missing kibana_discover_app_url for rule %s' % ( rule.get('name', '') ) ) @@ -79,7 +79,7 @@ def generate_kibana_discover_link(rule, match): return None return "%s?_g=%s&_a=%s" % ( - os.path.expandvars(discover_url), + os.path.expandvars(discover_app_url), urllib.parse.quote(globalState), urllib.parse.quote(appState) ) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 8a7294a6f..2190c05f1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -204,8 +204,8 @@ properties: scan_entire_timeframe: {type: boolean} ### Kibana Discover App Link - generate_kibana_discover_link: {type: boolean} - kibana_discover_url: {type: string} + generate_kibana_discover_url: {type: boolean} + kibana_discover_app_url: {type: string} kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} kibana_discover_columns: {type: array, items: {type: string}} diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index 8e5060d80..f06fe4e0c 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -2,14 +2,14 @@ from datetime import timedelta import pytest -from elastalert.kibana_discover import generate_kibana_discover_link +from elastalert.kibana_discover import generate_kibana_discover_url @pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -39,10 +39,10 @@ def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): @pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) -def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -71,10 +71,10 @@ def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): assert url == expectedUrl -def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_version(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_index_pattern_id': 'logs', 'timestamp_field': 'timestamp', 'name': 'test' @@ -86,8 +86,8 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): assert url is None -def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_app_url(): + url = generate_kibana_discover_url( rule={ 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs', @@ -101,10 +101,10 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): assert url is None -def test_generate_kibana_discover_link_with_missing_kibana_discover_index_pattern_id(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'timestamp_field': 'timestamp', 'name': 'test' @@ -116,10 +116,10 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_index_patter assert url is None -def test_generate_kibana_discover_link_with_invalid_kibana_version(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_invalid_kibana_version(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '4.5', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp' @@ -131,14 +131,14 @@ def test_generate_kibana_discover_link_with_invalid_kibana_version(): assert url is None -def test_generate_kibana_discover_link_with_discover_url_env_substitution(environ): +def test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitution(environ): environ.update({ 'KIBANA_HOST': 'kibana', 'KIBANA_PORT': '5601', }) - url = generate_kibana_discover_link( + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -167,10 +167,10 @@ def test_generate_kibana_discover_link_with_discover_url_env_substitution(enviro assert url == expectedUrl -def test_generate_kibana_discover_link_with_from_timedelta(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_from_timedelta(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), @@ -200,10 +200,10 @@ def test_generate_kibana_discover_link_with_from_timedelta(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), @@ -234,10 +234,10 @@ def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_to_timedelta(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_to_timedelta(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), @@ -267,10 +267,10 @@ def test_generate_kibana_discover_link_with_to_timedelta(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), @@ -301,10 +301,10 @@ def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), @@ -334,10 +334,10 @@ def test_generate_kibana_discover_link_with_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_custom_columns(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_custom_columns(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'kibana_discover_columns': ['level', 'message'], @@ -367,10 +367,10 @@ def test_generate_kibana_discover_link_with_custom_columns(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_single_filter(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_single_filter(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -418,10 +418,10 @@ def test_generate_kibana_discover_link_with_single_filter(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_multiple_filters(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_multiple_filters(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', 'timestamp_field': 'timestamp', @@ -472,10 +472,10 @@ def test_generate_kibana_discover_link_with_multiple_filters(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_int_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_int_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -532,10 +532,10 @@ def test_generate_kibana_discover_link_with_int_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_str_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_str_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -594,10 +594,10 @@ def test_generate_kibana_discover_link_with_str_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_null_query_key_value(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_null_query_key_value(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -644,10 +644,10 @@ def test_generate_kibana_discover_link_with_null_query_key_value(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_missing_query_key_value(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_query_key_value(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -693,10 +693,10 @@ def test_generate_kibana_discover_link_with_missing_query_key_value(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_compound_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_compound_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -781,10 +781,10 @@ def test_generate_kibana_discover_link_with_compound_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_filter_and_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_filter_and_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', From 53ab1ba18d9a40c6aa820fda1c7b2ea83218a126 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 14:48:01 -0400 Subject: [PATCH 64/74] Ensuring the version is a string (#5) --- elastalert/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 2190c05f1..62c2456b1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -206,7 +206,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string} - kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} kibana_discover_columns: {type: array, items: {type: string}} kibana_discover_from_timedelta: *timeframe From d719203ae0a10bea460b19884a5650bb7335d2ac Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 15:26:03 -0400 Subject: [PATCH 65/74] Upgrading jsonschema library to 3.0.2 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..39276ee3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 -jsonschema>=2.6.0 +jsonschema>=3.0.2 mock>=2.0.0 py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 diff --git a/setup.py b/setup.py index 412d53b1e..341b9fc7a 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=1.0.10,<1.0.15', - 'jsonschema>=2.6.0,<3.0.0', + 'jsonschema>=3.0.2', 'mock>=2.0.0', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', From 28e3ff15dc9dbcc6e55c747e72f6b2f69147965f Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 15:51:45 -0400 Subject: [PATCH 66/74] Updating to use draft 7 --- elastalert/loaders.py | 2 +- elastalert/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..0f28f0b69 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -90,7 +90,7 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml - self.rule_schema = jsonschema.Draft4Validator( + self.rule_schema = jsonschema.Draft7Validator( yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) self.base_config = copy.deepcopy(conf) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 13562c332..0135bdf55 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -1,4 +1,4 @@ -$schema: http://json-schema.org/draft-04/schema# +$schema: http://json-schema.org/draft-07/schema# definitions: # Either a single string OR an array of strings From 29b380e1497dbc3407e3189a02e1c1933fdb8eb9 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:05:42 -0400 Subject: [PATCH 67/74] Fixing the outstanding documentation warnings --- docs/source/recipes/writing_filters.rst | 2 +- docs/source/ruletypes.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index e923b89a5..1d2959262 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -57,7 +57,7 @@ a field that appears to have the value "foo bar", unless it is not analyzed. Con matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html `terms `_ -***** +***************************************************************************************************** diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ba43a187f..f7dc65337 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1305,7 +1305,7 @@ With ``alert_text_type: aggregation_summary_only``:: body = rule_name aggregation_summary -+ + ruletype_text is the string returned by RuleType.get_match_str. field_values will contain every key value pair included in the results from Elasticsearch. These fields include "@timestamp" (or the value of ``timestamp_field``), @@ -1689,7 +1689,7 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. Mattermost -~~~~~ +~~~~~~~~~~ Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters. From 9178b7d02d98fdecba259fe7f2172e75976b714e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:11:38 -0400 Subject: [PATCH 68/74] Removing empty header --- docs/source/elastalert.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 131d1742e..970cde7aa 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -203,7 +203,6 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``skip_invalid``: If ``True``, skip invalid files instead of exiting. -======= Logging ------- From 8ddfa7032d0bc580f416ef5b97e30bfab89afdb1 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:32:24 -0400 Subject: [PATCH 69/74] Treating documentation warnings as errors --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6a6efc293..71099e17c 100644 --- a/tox.ini +++ b/tox.ini @@ -27,4 +27,4 @@ norecursedirs = .* virtualenv_run docs build venv env deps = {[testenv]deps} sphinx==1.6.6 changedir = docs -commands = sphinx-build -b html -d build/doctrees source build/html +commands = sphinx-build -b html -d build/doctrees -W source build/html From 1392521d59fcfaafefce161b83fb64eb72218f9e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 17:59:55 -0400 Subject: [PATCH 70/74] Making schema more strict (#8) * Making schema more strict --- elastalert/schema.yaml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 62c2456b1..750563eee 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -11,6 +11,17 @@ definitions: type: [string, array] items: {type: [string, array]} + timedelta: &timedelta + type: object + additionalProperties: false + properties: + days: {type: number} + weeks: {type: number} + hours: {type: number} + minutes: {type: number} + seconds: {type: number} + milliseconds: {type: number} + timeFrame: &timeframe type: object additionalProperties: false @@ -205,12 +216,12 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} - kibana_discover_app_url: {type: string} + kibana_discover_app_url: {type: string, format: uri} kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} - kibana_discover_index_pattern_id: {type: string} - kibana_discover_columns: {type: array, items: {type: string}} - kibana_discover_from_timedelta: *timeframe - kibana_discover_to_timedelta: *timeframe + kibana_discover_index_pattern_id: {type: string, minLength: 1} + kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} + kibana_discover_from_timedelta: *timedelta + kibana_discover_to_timedelta: *timedelta # Alert Content alert_text: {type: string} # Python format string From db10b2ca4f1cfbc1302f1e843c13b15e4f8b845a Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 23 Sep 2019 14:55:35 -0700 Subject: [PATCH 71/74] Stop job when it gets disabled --- elastalert/elastalert.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5436ada18..e1c0dae92 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -23,8 +23,8 @@ from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException -from elasticsearch.exceptions import TransportError from elasticsearch.exceptions import NotFoundError +from elasticsearch.exceptions import TransportError from . import kibana from .alerts import DebugAlerter @@ -1067,7 +1067,13 @@ def load_rule_changes(self): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) - self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file] + for rule in self.rules: + if rule['rule_file'] == rule_file: + break + else: + continue + self.scheduler.remove_job(job_id=rule['name']) + self.rules.remove(rule) continue if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule From 3c0aa03945031cf55e7459bfa5631fcb46c3219b Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 23 Sep 2019 15:25:53 -0700 Subject: [PATCH 72/74] Added a test for rule scheduler removal --- tests/base_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/base_test.py b/tests/base_test.py index 15474c690..0e57a2ff9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1049,6 +1049,15 @@ def test_rule_changes(ea): ea.load_rule_changes() assert len(ea.rules) == 4 + # Disable a rule by removing the file + new_hashes.pop('rules/rule4.yaml') + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_hashes.return_value = new_hashes + ea.load_rule_changes() + ea.scheduler.remove_job.assert_called_with(job_id='rule4') + def test_strf_index(ea): """ Test that the get_index function properly generates indexes spanning days """ From a02b146dba509d852664d556e6e18435abf51dd9 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Wed, 25 Sep 2019 14:46:35 -0400 Subject: [PATCH 73/74] Adding ability to attach the Kibana Discover url as a seperate attachment in Slack notifications --- docs/source/ruletypes.rst | 6 ++ elastalert/alerts.py | 12 +++ elastalert/schema.yaml | 3 + tests/alerts_test.py | 200 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 49e2e5dea..3e0f78b9b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1781,6 +1781,12 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. +``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``slack_kibana_discover_color``: The color of the Kibana Discover url attachment. Defaults to ``#ec4b98``. + +``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. + Mattermost ~~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 9efa765c5..f5ca22070 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1129,6 +1129,9 @@ def __init__(self, rule): self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) self.slack_timeout = self.rule.get('slack_timeout', 10) self.slack_ca_certs = self.rule.get('slack_ca_certs') + self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) + self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') + self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') def format_body(self, body): # https://api.slack.com/docs/formatting @@ -1191,6 +1194,15 @@ def alert(self, matches): if self.slack_title_link != '': payload['attachments'][0]['title_link'] = self.slack_title_link + if self.slack_attach_kibana_discover_url: + kibana_discover_url = lookup_es_key(matches[0], 'kibana_discover_url') + if kibana_discover_url: + payload['attachments'].append({ + 'color': self.slack_kibana_discover_color, + 'title': self.slack_kibana_discover_title, + 'title_link': kibana_discover_url + }) + for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6cdad648e..afdffa20b 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -286,6 +286,9 @@ properties: slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} slack_ca_certs: {type: string} + slack_attach_kibana_discover_url {type: boolean} + slack_kibana_discover_color {type: string} + slack_kibana_discover_title {type: string} ### Mattermost mattermost_webhook_url: *arrayOfString diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 549b60624..da21f1b38 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1190,6 +1190,206 @@ def test_slack_uses_list_of_custom_slack_channel(): assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) +def test_slack_attach_kibana_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_attach_kibana_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_title': 'Click to discover in Kibana', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_color': 'blue', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', From 74ef682783b0d5ed185395f1198f3d4a7d37b0cb Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 26 Sep 2019 14:10:37 -0400 Subject: [PATCH 74/74] Adding ability to map match fields into opsgenie details --- docs/source/ruletypes.rst | 9 ++ elastalert/opsgenie.py | 23 +++- elastalert/schema.yaml | 14 +++ tests/alerts_test.py | 255 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 300 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 49e2e5dea..e9454c259 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1648,6 +1648,15 @@ Optional: ``opsgenie_priority``: Set the OpsGenie priority level. Possible values are P1, P2, P3, P4, P5. +``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. + +Example usage:: + + opsgenie_details: + Author: 'Bob Smith' # constant value + Environment: '$VAR' # environment variable + Message: { field: message } # field in the first match + SNS ~~~ diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index f984c03a0..bcdaf2d05 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import json import logging - +import os.path import requests from .alerts import Alerter @@ -33,6 +33,7 @@ def __init__(self, *args): self.alias = self.rule.get('opsgenie_alias') self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') + self.opsgenie_details = self.rule.get('opsgenie_details', {}) def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: @@ -97,6 +98,10 @@ def alert(self, matches): if self.alias is not None: post['alias'] = self.alias.format(**matches[0]) + details = self.get_details(matches) + if details: + post['details'] = details + logging.debug(json.dumps(post)) headers = { @@ -162,3 +167,19 @@ def get_info(self): if self.teams: ret['teams'] = self.teams return ret + + def get_details(self, matches): + details = {} + + for key, value in self.opsgenie_details.items(): + + if type(value) is dict: + if 'field' in value: + field_value = lookup_es_key(matches[0], value['field']) + if field_value is not None: + details[key] = str(field_value) + + elif type(value) is str: + details[key] = os.path.expandvars(value) + + return details diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6cdad648e..b940acd47 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -298,6 +298,20 @@ properties: mattermost_msg_pretext: {type: string} mattermost_msg_fields: *mattermostField + ## Opsgenie + opsgenie_details: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + ### PagerDuty pagerduty_service_key: {type: string} pagerduty_client_name: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 549b60624..e3a897e5f 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -456,6 +456,261 @@ def test_opsgenie_default_alert_routing(): assert alert.get_info()['recipients'] == ['devops@test.com'] +def test_opsgenie_details_with_constant_value(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': 'Bar'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'message'}} + } + match = { + 'message': 'Bar', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_nested_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'nested.field'}} + } + match = { + 'nested': { + 'field': 'Bar' + }, + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_non_string_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Age': {'field': 'age'}, + 'Message': {'field': 'message'} + } + } + match = { + 'age': 10, + 'message': { + 'format': 'The cow goes %s!', + 'arg0': 'moo' + } + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': { + 'Age': '10', + 'Message': "{'format': 'The cow goes %s!', 'arg0': 'moo'}" + }, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_missing_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Message': {'field': 'message'}, + 'Missing': {'field': 'missing'} + } + } + match = { + 'message': 'Testing', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Message': 'Testing'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_environment_variable_replacement(environ): + environ.update({ + 'TEST_VAR': 'Bar' + }) + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': '$TEST_VAR'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = {