diff --git a/.travis.yml b/.travis.yml index 7c8ba2b34..569bf12d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: -- '2.7' +- '3.6' env: - TOXENV=docs -- TOXENV=py27 +- TOXENV=py36 install: - pip install tox - > @@ -19,16 +19,16 @@ script: make test-elasticsearch else make test - fi + fi jobs: include: - stage: 'Elasticsearch test' - env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64 - - env: TOXENV=py27 ES_VERSION=6.6.2 - - env: TOXENV=py27 ES_VERSION=6.3.2 - - env: TOXENV=py27 ES_VERSION=6.2.4 - - env: TOXENV=py27 ES_VERSION=6.0.1 - - env: TOXENV=py27 ES_VERSION=5.6.16 + env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 + - env: TOXENV=py36 ES_VERSION=6.6.2 + - env: TOXENV=py36 ES_VERSION=6.3.2 + - env: TOXENV=py36 ES_VERSION=6.2.4 + - env: TOXENV=py36 ES_VERSION=6.0.1 + - env: TOXENV=py36 ES_VERSION=5.6.16 deploy: provider: pypi diff --git a/Dockerfile-test b/Dockerfile-test index 761a777c6..3c153e644 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,9 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip +RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert ADD requirements*.txt ./ -RUN pip install -r requirements-dev.txt +RUN pip3 install -r requirements-dev.txt diff --git a/README.md b/README.md index 8757290f6..99acc02e7 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ -Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` +Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. - -[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) -[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -74,8 +71,23 @@ In addition to this basic usage, there are many other features that make alerts To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). ## Running ElastAlert +You can either install the latest released version of ElastAlert using pip: + +```pip install elastalert``` + +or you can clone the ElastAlert repository for the most recent changes: + +```git clone https://github.com/Yelp/elastalert.git``` + +Install the module: + +```pip install "setuptools>=11.3"``` + +```python setup.py install``` + +The following invocation can be used to run ElastAlert after installing -``$ python elastalert/elastalert.py [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` +``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` ``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. diff --git a/changelog.md b/changelog.md index 30b52f601..fe30b573c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,22 @@ # Change Log +# v0.2.1 + +### Fixed +- Fixed an AttributeError introduced in 0.2.0 + +# v0.2.0 + +- Switched to Python 3 + +### Added +- Add rule loader class for customized rule loading +- Added thread based rules and limit_execution +- Run_every can now be customized per rule + +### Fixed +- Various small fixes + # v0.1.39 ### Added diff --git a/config.yaml.example b/config.yaml.example index e39a2bcf8..9d9176382 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,6 +57,7 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 12a07fe31..970cde7aa 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -133,9 +133,12 @@ The environment variable ``ES_USE_SSL`` will override this field. ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``. +``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes. +Defaults to ``FileRulesLoader`` if not set. + ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload -or remove rules based on their respective config files. +or remove rules based on their respective config files. (only required when using ``FileRulesLoader``). ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true`` @@ -200,7 +203,6 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``skip_invalid``: If ``True``, skip invalid files instead of exiting. -======= Logging ------- diff --git a/docs/source/index.rst b/docs/source/index.rst index c3f02f535..4219bf13e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: recipes/adding_alerts recipes/writing_filters recipes/adding_enhancements + recipes/adding_loaders recipes/signing_requests Indices and Tables diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst new file mode 100644 index 000000000..672d42390 --- /dev/null +++ b/docs/source/recipes/adding_loaders.rst @@ -0,0 +1,85 @@ +.. _loaders: + +Rules Loaders +======================== + +RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to +gather rules for a particular source. Your RulesLoader needs to implement three member functions, and +will look something like this: + +.. code-block:: python + + class AwesomeNewRulesLoader(RulesLoader): + def get_names(self, conf, use_rule=None): + ... + def get_hashes(self, conf, use_rule=None): + ... + def get_yaml(self, rule): + ... + +You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a +python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``. + +Example +------- + +As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a +modules folder for the loader in the ElastAlert directory. + +.. code-block:: console + + $ mkdir elastalert_modules + $ cd elastalert_modules + $ touch __init__.py + +Now, in a file named ``mongo_loader.py``, add + +.. code-block:: python + + from pymongo import MongoClient + from elastalert.loaders import RulesLoader + import yaml + + class MongoRulesLoader(RulesLoader): + def __init__(self, conf): + super(MongoRulesLoader, self).__init__(conf) + self.client = MongoClient(conf['mongo_url']) + self.db = self.client[conf['mongo_db']] + self.cache = {} + + def get_names(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + rules = [] + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule['name']] = yaml.load(rule['yaml']) + rules.append(rule['name']) + + return rules + + def get_hashes(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + hashes = {} + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule['name']] = rule['yaml'] + hashes[rule['name']] = rule['hash'] + + return hashes + + def get_yaml(self, rule): + if rule in self.cache: + return self.cache[rule] + + self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml']) + return self.cache[rule] + +Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the +default FileRulesLoader, so in your ``elastalert.conf`` file:: + + rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader" + diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index e923b89a5..1d2959262 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -57,7 +57,7 @@ a field that appears to have the value "foo bar", unless it is not analyzed. Con matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html `terms `_ -***** +***************************************************************************************************** diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f8452dcdd..8d9644060 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,6 +58,20 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | +| ``generate_kibana_discover_url`` (boolean, default False) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_app_url`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_version`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_index_pattern_id`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_columns`` (list of strs, default _source) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_from_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_to_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | | ``use_local_time`` (boolean, default True) | | +--------------------------------------------------------------+ | | ``realert`` (time, default: 1 min) | | @@ -510,6 +524,85 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` +generate_kibana_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application. +This setting requires the following settings are also configured: + +- ``kibana_discover_app_url`` +- ``kibana_discover_version`` +- ``kibana_discover_index_pattern_id`` + +``generate_kibana_discover_url: true`` + +kibana_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. + +``kibana_discover_app_url: http://kibana:5601/#/discover`` + +kibana_discover_version +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_version``: Specifies the version of the Kibana Discover application. + +The currently supported versions of Kibana Discover are: + +- `5.6` +- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` +- `7.0`, `7.1`, `7.2`, `7.3` + +``kibana_discover_version: '7.3'`` + +kibana_discover_index_pattern_id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. +These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. + +Example export of an index pattern's saved object: + +.. code-block:: text + + [ + { + "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c", + "_type": "index-pattern", + "_source": { ... } + } + ] + +You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing. + +``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` + +kibana_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. +Defaults to the ``_source`` column. + +``kibana_discover_columns: [ timestamp, message ]`` + +kibana_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``kibana_discover_from_timedelta: minutes: 2`` + +kibana_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``kibana_discover_to_timedelta: minutes: 2`` + use_local_time ^^^^^^^^^^^^^^ @@ -1305,7 +1398,7 @@ With ``alert_text_type: aggregation_summary_only``:: body = rule_name aggregation_summary -+ + ruletype_text is the string returned by RuleType.get_match_str. field_values will contain every key value pair included in the results from Elasticsearch. These fields include "@timestamp" (or the value of ``timestamp_field``), @@ -1555,6 +1648,15 @@ Optional: ``opsgenie_priority``: Set the OpsGenie priority level. Possible values are P1, P2, P3, P4, P5. +``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. + +Example usage:: + + opsgenie_details: + Author: 'Bob Smith' # constant value + Environment: '$VAR' # environment variable + Message: { field: message } # field in the first match + SNS ~~~ @@ -1688,8 +1790,14 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. +``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``slack_kibana_discover_color``: The color of the Kibana Discover url attachment. Defaults to ``#ec4b98``. + +``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. + Mattermost -~~~~~ +~~~~~~~~~~ Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters. @@ -2037,6 +2145,8 @@ Optional: ``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here. +``http_post_headers``: Key:value pairs of headers to be sent as part of the request. + ``http_post_proxy``: URL of proxy, if required. ``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False. @@ -2051,6 +2161,8 @@ Example usage:: ip: clientip http_post_static_payload: apikey: abc123 + http_post_headers: + authorization: Basic 123dr3234 Alerter diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 09e307c24..7fdf1eeba 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -8,7 +8,7 @@ Requirements - Elasticsearch - ISO8601 or Unix timestamped data -- Python 2.7 +- Python 3.6 - pip, see requirements.txt - Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 90b14126e..55bfdb32f 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- import copy -import logging -from elasticsearch import Elasticsearch, RequestsHttpConnection -from elasticsearch.client import query_params, _make_path +import time + +from elasticsearch import Elasticsearch +from elasticsearch import RequestsHttpConnection +from elasticsearch.client import _make_path +from elasticsearch.client import query_params +from elasticsearch.exceptions import TransportError class ElasticSearchClient(Elasticsearch): @@ -40,7 +44,14 @@ def es_version(self): Returns the reported version from the Elasticsearch server. """ if self._es_version is None: - self._es_version = self.info()['version']['number'] + for retry in range(3): + try: + self._es_version = self.info()['version']['number'] + break + except TransportError: + if retry == 2: + raise + time.sleep(3) return self._es_version def is_atleastfive(self): @@ -59,14 +70,14 @@ def is_atleastsixtwo(self): """ Returns True when the Elasticsearch server version >= 6.2 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) def is_atleastsixsix(self): """ Returns True when the Elasticsearch server version >= 6.6 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 6) def is_atleastseven(self): @@ -232,14 +243,15 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): :arg version: Specify whether to return document version as part of a hit """ - logging.warning( - 'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8') # from is a reserved word so it cannot be used, use from_ instead if "from_" in params: params["from"] = params.pop("from_") if not index: index = "_all" - return self.transport.perform_request( + res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body ) + if type(res) == list or type(res) == tuple: + return res[1] + return res diff --git a/elastalert/alerts.py b/elastalert/alerts.py index bf0d356f8..4fc1f3661 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -11,7 +11,7 @@ import warnings from email.mime.text import MIMEText from email.utils import formatdate -from HTMLParser import HTMLParser +from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError @@ -34,13 +34,14 @@ from thehive4py.models import CustomFieldHelper from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient -from util import EAException -from util import elastalert_logger -from util import lookup_es_key -from util import pretty_ts -from util import resolve_string -from util import ts_now -from util import ts_to_dt + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import pretty_ts +from .util import resolve_string +from .util import ts_now +from .util import ts_to_dt class DateTimeEncoder(json.JSONEncoder): @@ -71,7 +72,6 @@ def _create_custom_alert_text(self, event=None, alert_text_key='alert_text'): if alert_text_key+'_args' in self.rule: alert_text_args = self.rule.get(alert_text_key+'_args') alert_text_values = [lookup_es_key(event, arg) for arg in alert_text_args] - # Support referencing other top-level rule properties # This technically may not work if there is a top-level rule property with the same name # as an es result key, since it would have been matched in the lookup_es_key call above @@ -87,7 +87,6 @@ def _create_custom_alert_text(self, event=None, alert_text_key='alert_text'): kw = {} for name, kw_name in self.rule.get(alert_text_key+'_kw').items(): val = lookup_es_key(event, name) - # Support referencing other top-level rule properties # This technically may not work if there is a top-level rule property with the same name # as an es result key, since it would have been matched in the lookup_es_key call above @@ -107,10 +106,10 @@ def _add_rule_text(self): self.text += self.rule['type'].get_match_str(self.match) def _add_top_counts(self): - for key, counts in self.match.items(): + for key, counts in list(self.match.items()): if key.startswith('top_events_'): self.text += '%s:\n' % (key[11:]) - top_events = counts.items() + top_events = list(counts.items()) if not top_events: self.text += 'No events found.\n' @@ -122,12 +121,12 @@ def _add_top_counts(self): self.text += '\n' def _add_match_items(self): - match_items = self.match.items() + match_items = list(self.match.items()) match_items.sort(key=lambda x: x[0]) for key, value in match_items: if key.startswith('top_events_'): continue - value_str = unicode(value) + value_str = str(value) value_str.replace('\\n', '\n') if type(value) in [list, dict]: try: @@ -181,9 +180,9 @@ def __str__(self): class JiraFormattedMatchString(BasicMatchString): def _add_match_items(self): - match_items = dict([(x, y) for x, y in self.match.items() if not x.startswith('top_events_')]) + match_items = dict([(x, y) for x, y in list(self.match.items()) if not x.startswith('top_events_')]) json_blob = self._pretty_print_as_json(match_items) - preformatted_text = u'{{code}}{0}{{code}}'.format(json_blob) + preformatted_text = '{{code}}{0}{{code}}'.format(json_blob) self.text += preformatted_text @@ -212,14 +211,14 @@ def resolve_rule_references(self, root): root[i] = self.resolve_rule_reference(item) elif type(root) == dict: # Make a copy since we may be modifying the contents of the structure we're walking - for key, value in root.copy().iteritems(): + for key, value in root.copy().items(): if type(value) == dict or type(value) == list: self.resolve_rule_references(root[key]) else: root[key] = self.resolve_rule_reference(value) def resolve_rule_reference(self, value): - strValue = unicode(value) + strValue = str(value) if strValue.startswith('$') and strValue.endswith('$') and strValue[1:-1] in self.rule: if type(value) == int: return int(self.rule[strValue[1:-1]]) @@ -251,7 +250,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - alert_subject = unicode(self.rule['alert_subject']) + alert_subject = str(self.rule['alert_subject']) alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048)) if 'alert_subject_args' in self.rule: @@ -280,7 +279,7 @@ def create_alert_body(self, matches): body = self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -310,16 +309,16 @@ def get_aggregation_summary_text(self, matches): # Maintain an aggregate count for each unique key encountered in the aggregation period for match in matches: - key_tuple = tuple([unicode(lookup_es_key(match, key)) for key in summary_table_fields]) + key_tuple = tuple([str(lookup_es_key(match, key)) for key in summary_table_fields]) if key_tuple not in match_aggregation: match_aggregation[key_tuple] = 1 else: match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1 - for keys, count in match_aggregation.iteritems(): + for keys, count in match_aggregation.items(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' text += self.rule.get('summary_prefix', '') - return unicode(text) + return str(text) def create_default_title(self, matches): return self.rule['name'] @@ -375,13 +374,13 @@ def alert(self, matches): ) fullmessage['match'] = lookup_es_key( match, self.rule['timestamp_field']) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) fullmessage['alerts'] = alerts fullmessage['rule'] = self.rule['name'] fullmessage['rule_file'] = self.rule['rule_file'] - fullmessage['matching'] = unicode(BasicMatchString(self.rule, match)) + fullmessage['matching'] = str(BasicMatchString(self.rule, match)) fullmessage['alertDate'] = datetime.datetime.now( ).strftime("%Y-%m-%d %H:%M:%S") fullmessage['body'] = self.create_alert_body(matches) @@ -420,7 +419,7 @@ def alert(self, matches): 'Alert for %s, %s at %s:' % (self.rule['name'], match[qk], lookup_es_key(match, self.rule['timestamp_field']))) else: elastalert_logger.info('Alert for %s at %s:' % (self.rule['name'], lookup_es_key(match, self.rule['timestamp_field']))) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) def get_info(self): return {'type': 'debug'} @@ -442,15 +441,15 @@ def __init__(self, *args): self.smtp_key_file = self.rule.get('smtp_key_file') self.smtp_cert_file = self.rule.get('smtp_cert_file') # Convert email to a list if it isn't already - if isinstance(self.rule['email'], basestring): + if isinstance(self.rule['email'], str): self.rule['email'] = [self.rule['email']] # If there is a cc then also convert it a list if it isn't cc = self.rule.get('cc') - if cc and isinstance(cc, basestring): + if cc and isinstance(cc, str): self.rule['cc'] = [self.rule['cc']] # If there is a bcc then also convert it to a list if it isn't bcc = self.rule.get('bcc') - if bcc and isinstance(bcc, basestring): + if bcc and isinstance(bcc, str): self.rule['bcc'] = [self.rule['bcc']] add_suffix = self.rule.get('email_add_domain') if add_suffix and not add_suffix.startswith('@'): @@ -467,7 +466,7 @@ def alert(self, matches): to_addr = self.rule['email'] if 'email_from_field' in self.rule: recipient = lookup_es_key(matches[0], self.rule['email_from_field']) - if isinstance(recipient, basestring): + if isinstance(recipient, str): if '@' in recipient: to_addr = [recipient] elif 'email_add_domain' in self.rule: @@ -477,9 +476,9 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - email_msg = MIMEText(body.encode('UTF-8'), 'html', _charset='UTF-8') + email_msg = MIMEText(body, 'html', _charset='UTF-8') else: - email_msg = MIMEText(body.encode('UTF-8'), _charset='UTF-8') + email_msg = MIMEText(body, _charset='UTF-8') email_msg['Subject'] = self.create_title(matches) email_msg['To'] = ', '.join(to_addr) email_msg['From'] = self.from_addr @@ -624,7 +623,7 @@ def __init__(self, rule): self.get_arbitrary_fields() except JIRAError as e: # JIRAError may contain HTML, pass along only first 1024 chars - raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])), None, sys.exc_info()[2] + raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])).with_traceback(sys.exc_info()[2]) self.set_priority() @@ -633,7 +632,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, self.priority_ids.keys())) + logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -727,7 +726,7 @@ def get_arbitrary_fields(self): # Clear jira_args self.reset_jira_args() - for jira_field, value in self.rule.iteritems(): + for jira_field, value in self.rule.items(): # If we find a field that is not covered by the set that we are aware of, it means it is either: # 1. A built-in supported field in JIRA that we don't have on our radar # 2. A custom field that a JIRA admin has configured @@ -783,7 +782,7 @@ def find_existing_ticket(self, matches): return issues[0] def comment_on_ticket(self, ticket, match): - text = unicode(JiraFormattedMatchString(self.rule, match)) + text = str(JiraFormattedMatchString(self.rule, match)) timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field'])) comment = "This alert was triggered again at %s\n%s" % (timestamp, text) self.client.add_comment(ticket, comment) @@ -858,7 +857,7 @@ def alert(self, matches): "Exception encountered when trying to add '{0}' as a watcher. Does the user exist?\n{1}" .format( watcher, ex - )), None, sys.exc_info()[2] + )).with_traceback(sys.exc_info()[2]) except JIRAError as e: raise EAException("Error creating JIRA ticket using jira_args (%s): %s" % (self.jira_args, e)) @@ -873,7 +872,7 @@ def create_alert_body(self, matches): body += self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(JiraFormattedMatchString(self.rule, match)) + body += str(JiraFormattedMatchString(self.rule, match)) if len(matches) > 1: body += '\n----------------------------------------\n' return body @@ -881,7 +880,7 @@ def create_alert_body(self, matches): def get_aggregation_summary_text(self, matches): text = super(JiraAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'{{noformat}}{0}{{noformat}}'.format(text) + text = '{{noformat}}{0}{{noformat}}'.format(text) return text def create_default_title(self, matches, for_search=False): @@ -917,7 +916,7 @@ def __init__(self, *args): self.last_command = [] self.shell = False - if isinstance(self.rule['command'], basestring): + if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: logging.warning('Warning! You could be vulnerable to shell injection!') @@ -941,10 +940,10 @@ def alert(self, matches): if self.rule.get('pipe_match_json'): match_json = json.dumps(matches, cls=DateTimeEncoder) + '\n' - stdout, stderr = subp.communicate(input=match_json) + stdout, stderr = subp.communicate(input=match_json.encode()) elif self.rule.get('pipe_alert_text'): alert_text = self.create_alert_body(matches) - stdout, stderr = subp.communicate(input=alert_text) + stdout, stderr = subp.communicate(input=alert_text.encode()) if self.rule.get("fail_on_non_zero_exit", False) and subp.wait(): raise EAException("Non-zero exit code while running command %s" % (' '.join(command))) except OSError as e: @@ -1083,7 +1082,7 @@ class MsTeamsAlerter(Alerter): def __init__(self, rule): super(MsTeamsAlerter, self).__init__(rule) self.ms_teams_webhook_url = self.rule['ms_teams_webhook_url'] - if isinstance(self.ms_teams_webhook_url, basestring): + if isinstance(self.ms_teams_webhook_url, str): self.ms_teams_webhook_url = [self.ms_teams_webhook_url] self.ms_teams_proxy = self.rule.get('ms_teams_proxy', None) self.ms_teams_alert_summary = self.rule.get('ms_teams_alert_summary', 'ElastAlert Message') @@ -1091,7 +1090,6 @@ def __init__(self, rule): self.ms_teams_theme_color = self.rule.get('ms_teams_theme_color', '') def format_body(self, body): - body = body.encode('UTF-8') if self.ms_teams_alert_fixed_width: body = body.replace('`', "'") body = "```{0}```".format('```\n\n```'.join(x for x in body.split('\n'))).replace('\n``````', '') @@ -1135,12 +1133,12 @@ class SlackAlerter(Alerter): def __init__(self, rule): super(SlackAlerter, self).__init__(rule) self.slack_webhook_url = self.rule['slack_webhook_url'] - if isinstance(self.slack_webhook_url, basestring): + if isinstance(self.slack_webhook_url, str): self.slack_webhook_url = [self.slack_webhook_url] self.slack_proxy = self.rule.get('slack_proxy', None) self.slack_username_override = self.rule.get('slack_username_override', 'elastalert') self.slack_channel_override = self.rule.get('slack_channel_override', '') - if isinstance(self.slack_channel_override, basestring): + if isinstance(self.slack_channel_override, str): self.slack_channel_override = [self.slack_channel_override] self.slack_title_link = self.rule.get('slack_title_link', '') self.slack_title = self.rule.get('slack_title', '') @@ -1152,10 +1150,14 @@ def __init__(self, rule): self.slack_alert_fields = self.rule.get('slack_alert_fields', '') self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) self.slack_timeout = self.rule.get('slack_timeout', 10) + self.slack_ca_certs = self.rule.get('slack_ca_certs') + self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) + self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') + self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') def format_body(self, body): # https://api.slack.com/docs/formatting - return body.encode('UTF-8') + return body def get_aggregation_summary_text__maximum_width(self): width = super(SlackAlerter, self).get_aggregation_summary_text__maximum_width() @@ -1165,7 +1167,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(SlackAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1214,15 +1216,28 @@ def alert(self, matches): if self.slack_title_link != '': payload['attachments'][0]['title_link'] = self.slack_title_link + if self.slack_attach_kibana_discover_url: + kibana_discover_url = lookup_es_key(matches[0], 'kibana_discover_url') + if kibana_discover_url: + payload['attachments'].append({ + 'color': self.slack_kibana_discover_color, + 'title': self.slack_kibana_discover_title, + 'title_link': kibana_discover_url + }) + for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: + if self.slack_ca_certs: + verify = self.slack_ca_certs + else: + verify = self.slack_ignore_ssl_errors if self.slack_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() payload['channel'] = channel_override response = requests.post( url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.slack_ignore_ssl_errors, + headers=headers, verify=verify, proxies=proxies, timeout=self.slack_timeout) warnings.resetwarnings() @@ -1245,7 +1260,7 @@ def __init__(self, rule): # HTTP config self.mattermost_webhook_url = self.rule['mattermost_webhook_url'] - if isinstance(self.mattermost_webhook_url, basestring): + if isinstance(self.mattermost_webhook_url, str): self.mattermost_webhook_url = [self.mattermost_webhook_url] self.mattermost_proxy = self.rule.get('mattermost_proxy', None) self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False) @@ -1268,7 +1283,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(MattermostAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1401,7 +1416,7 @@ def alert(self, matches): matches), 'summary': self.create_title(matches), 'custom_details': { - 'information': body.encode('UTF-8'), + 'information': body, }, }, } @@ -1416,7 +1431,7 @@ def alert(self, matches): 'incident_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'details': { - "information": body.encode('UTF-8'), + "information": body, }, } @@ -1532,7 +1547,7 @@ def alert(self, matches): if response != 200: raise EAException("Error posting to Exotel, response code is %s" % response) except RequestException: - raise EAException("Error posting to Exotel"), None, sys.exc_info()[2] + raise EAException("Error posting to Exotel").with_traceback(sys.exc_info()[2]) elastalert_logger.info("Trigger sent to Exotel") def get_info(self): @@ -1625,15 +1640,15 @@ def __init__(self, rule): self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None) def alert(self, matches): - body = u'⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) + body = '⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' if len(body) > 4095: - body = body[0:4000] + u"\n⚠ *message was cropped according to telegram limits!* ⚠" - body += u' ```' + body = body[0:4000] + "\n⚠ *message was cropped according to telegram limits!* ⚠" + body += ' ```' headers = {'content-type': 'application/json'} # set https proxy, if it was provided @@ -1668,7 +1683,7 @@ class GoogleChatAlerter(Alerter): def __init__(self, rule): super(GoogleChatAlerter, self).__init__(rule) self.googlechat_webhook_url = self.rule['googlechat_webhook_url'] - if isinstance(self.googlechat_webhook_url, basestring): + if isinstance(self.googlechat_webhook_url, str): self.googlechat_webhook_url = [self.googlechat_webhook_url] self.googlechat_format = self.rule.get('googlechat_format', 'basic') self.googlechat_header_title = self.rule.get('googlechat_header_title', None) @@ -1708,7 +1723,7 @@ def create_card(self, matches): card = {"cards": [{ "sections": [{ "widgets": [ - {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} + {"textParagraph": {"text": self.create_alert_body(matches)}} ]} ]} ]} @@ -1726,7 +1741,6 @@ def create_card(self, matches): def create_basic(self, matches): body = self.create_alert_body(matches) - body = body.encode('UTF-8') return {'text': body} def alert(self, matches): @@ -1884,7 +1898,6 @@ def alert(self, matches): headers = {'content-type': 'application/json'} if self.api_key is not None: headers['Authorization'] = 'Key %s' % (self.rule['alerta_api_key']) - alerta_payload = self.get_json_payload(matches[0]) try: @@ -1943,8 +1956,8 @@ def get_json_payload(self, match): 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], - 'attributes': dict(zip(self.attributes_keys, - [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), + 'attributes': dict(list(zip(self.attributes_keys, + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values]))), 'rawData': self.create_alert_body([match]), } @@ -1961,7 +1974,7 @@ class HTTPPostAlerter(Alerter): def __init__(self, rule): super(HTTPPostAlerter, self).__init__(rule) post_url = self.rule.get('http_post_url') - if isinstance(post_url, basestring): + if isinstance(post_url, str): post_url = [post_url] self.post_url = post_url self.post_proxy = self.rule.get('http_post_proxy') @@ -1976,7 +1989,7 @@ def alert(self, matches): for match in matches: payload = match if self.post_all_values else {} payload.update(self.post_static_payload) - for post_key, es_key in self.post_payload.items(): + for post_key, es_key in list(self.post_payload.items()): payload[post_key] = lookup_es_key(match, es_key) headers = { "Content-Type": "application/json", @@ -2182,7 +2195,7 @@ def send_to_thehive(self, alert_config): connection_details = self.rule['hive_connection'] api = TheHiveApi( - '{hive_host}:{hive_port}'.format(**connection_details), + connection_details.get('hive_host'), connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) @@ -2201,6 +2214,7 @@ def alert(self, matches): else: alert_config = self.create_alert_config(matches[0]) artifacts = [] + for match in matches: artifacts += self.create_artifacts(match) if 'related_events' in match: diff --git a/elastalert/config.py b/elastalert/config.py index 7b7fc30ba..5ae9a26e6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,38 +1,18 @@ # -*- coding: utf-8 -*- -import copy import datetime -import hashlib import logging import logging.config -import os -import sys -import alerts -import enhancements -import jsonschema -import ruletypes -import yaml -import yaml.scanner from envparse import Env -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms -from util import elastalert_logger -from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt -# schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) +from . import loaders +from .util import EAException +from .util import elastalert_logger +from .util import get_module -# Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) -required_locals = frozenset(['alert', 'type', 'name', 'index']) +# Required global (config.yaml) configuration options +required_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -44,440 +24,57 @@ env = Env(ES_USE_SSL=bool) -# import rule dependency -import_rules = {} -# Used to map the names of rules to their classes -rules_mapping = { - 'frequency': ruletypes.FrequencyRule, - 'any': ruletypes.AnyRule, - 'spike': ruletypes.SpikeRule, - 'blacklist': ruletypes.BlacklistRule, - 'whitelist': ruletypes.WhitelistRule, - 'change': ruletypes.ChangeRule, - 'flatline': ruletypes.FlatlineRule, - 'new_term': ruletypes.NewTermsRule, - 'cardinality': ruletypes.CardinalityRule, - 'metric_aggregation': ruletypes.MetricAggregationRule, - 'percentage_match': ruletypes.PercentageMatchRule, - 'spike_aggregation': ruletypes.SpikeMetricAggregationRule +# Used to map the names of rule loaders to their classes +loader_mapping = { + 'file': loaders.FileRulesLoader, } -# Used to map names of alerts to their classes -alerts_mapping = { - 'email': alerts.EmailAlerter, - 'jira': alerts.JiraAlerter, - 'opsgenie': OpsGenieAlerter, - 'stomp': alerts.StompAlerter, - 'debug': alerts.DebugAlerter, - 'command': alerts.CommandAlerter, - 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, - 'ms_teams': alerts.MsTeamsAlerter, - 'slack': alerts.SlackAlerter, - 'mattermost': alerts.MattermostAlerter, - 'pagerduty': alerts.PagerDutyAlerter, - 'pagertree': alerts.PagerTreeAlerter, - 'exotel': alerts.ExotelAlerter, - 'twilio': alerts.TwilioAlerter, - 'victorops': alerts.VictorOpsAlerter, - 'telegram': alerts.TelegramAlerter, - 'googlechat': alerts.GoogleChatAlerter, - 'gitter': alerts.GitterAlerter, - 'servicenow': alerts.ServiceNowAlerter, - 'linenotify': alerts.LineNotifyAlerter, - 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter, - 'hivealerter': alerts.HiveAlerter -} -# A partial ordering of alert types. Relative order will be preserved in the resulting alerts list -# For example, jira goes before email so the ticket # will be added to the resulting email. -alerts_order = { - 'jira': 0, - 'email': 1 -} - -base_config = {} - - -def get_module(module_name): - """ Loads a module and returns a specific object. - module_name should 'module.file.object'. - Returns object or raises EAException on error. """ - try: - module_path, module_class = module_name.rsplit('.', 1) - base_module = __import__(module_path, globals(), locals(), [module_class]) - module = getattr(base_module, module_class) - except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] - return module - - -def load_configuration(filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :return: The rule configuration, a dictionary. - """ - try: - rule = load_rule_yaml(filename) - except Exception as e: - if (conf.get('skip_invalid')): - logging.error(e) - return False - else: - raise e - load_options(rule, conf, filename, args) - load_modules(rule, args) - return rule - - -def load_rule_yaml(filename): - rule = { - 'rule_file': filename, - } - - import_rules.pop(filename, None) # clear `filename` dependency - while True: - try: - loaded = yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - - # Special case for merging filters - if both files specify a filter merge (AND) them - if 'filter' in rule and 'filter' in loaded: - rule['filter'] = loaded['filter'] + rule['filter'] - - loaded.update(rule) - rule = loaded - if 'import' in rule: - # Find the path of the next file. - if os.path.isabs(rule['import']): - import_filename = rule['import'] - else: - import_filename = os.path.join(os.path.dirname(filename), rule['import']) - # set dependencies - rules = import_rules.get(filename, []) - rules.append(import_filename) - import_rules[filename] = rules - filename = import_filename - del(rule['import']) # or we could go on forever! - else: - break - - return rule - - -def load_options(rule, conf, filename, args=None): - """ Converts time objects, sets defaults, and validates some settings. - - :param rule: A dictionary of parsed YAML from a rule config file. - :param conf: The global configuration dictionary, used for populating defaults. - """ - adjust_deprecated_values(rule) - - try: - rule_schema.validate(rule) - except jsonschema.ValidationError as e: - raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) - - try: - # Set all time based parameters - if 'timeframe' in rule: - rule['timeframe'] = datetime.timedelta(**rule['timeframe']) - if 'realert' in rule: - rule['realert'] = datetime.timedelta(**rule['realert']) - else: - if 'aggregation' in rule: - rule['realert'] = datetime.timedelta(minutes=0) - else: - rule['realert'] = datetime.timedelta(minutes=1) - if 'aggregation' in rule and not rule['aggregation'].get('schedule'): - rule['aggregation'] = datetime.timedelta(**rule['aggregation']) - if 'query_delay' in rule: - rule['query_delay'] = datetime.timedelta(**rule['query_delay']) - if 'buffer_time' in rule: - rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) - if 'bucket_interval' in rule: - rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) - if 'exponential_realert' in rule: - rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) - if 'kibana4_start_timedelta' in rule: - rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) - if 'kibana4_end_timedelta' in rule: - rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) - except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) - - # Set defaults, copy defaults from config.yaml - td_fields = ['realert', 'exponential_realert', 'aggregation', 'query_delay'] - for td_field in td_fields: - if td_field in base_config: - rule.setdefault(td_field, datetime.timedelta(**base_config[td_field])) - for key, val in base_config.items(): - rule.setdefault(key, val) - rule.setdefault('name', os.path.splitext(filename)[0]) - rule.setdefault('realert', datetime.timedelta(seconds=0)) - rule.setdefault('aggregation', datetime.timedelta(seconds=0)) - rule.setdefault('query_delay', datetime.timedelta(seconds=0)) - rule.setdefault('timestamp_field', '@timestamp') - rule.setdefault('filter', []) - rule.setdefault('timestamp_type', 'iso') - rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') - rule.setdefault('_source_enabled', True) - rule.setdefault('use_local_time', True) - rule.setdefault('description', "") - - # Set timestamp_type conversion function, used when generating queries and processing hits - rule['timestamp_type'] = rule['timestamp_type'].strip().lower() - if rule['timestamp_type'] == 'iso': - rule['ts_to_dt'] = ts_to_dt - rule['dt_to_ts'] = dt_to_ts - elif rule['timestamp_type'] == 'unix': - rule['ts_to_dt'] = unix_to_dt - rule['dt_to_ts'] = dt_to_unix - elif rule['timestamp_type'] == 'unix_ms': - rule['ts_to_dt'] = unixms_to_dt - rule['dt_to_ts'] = dt_to_unixms - elif rule['timestamp_type'] == 'custom': - def _ts_to_dt_with_format(ts): - return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) - - def _dt_to_ts_with_format(dt): - ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) - if 'timestamp_format_expr' in rule: - # eval expression passing 'ts' and 'dt' - return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) - else: - return ts - - rule['ts_to_dt'] = _ts_to_dt_with_format - rule['dt_to_ts'] = _dt_to_ts_with_format - else: - raise EAException('timestamp_type must be one of iso, unix, or unix_ms') - - # Add support for client ssl certificate auth - if 'verify_certs' in conf: - rule.setdefault('verify_certs', conf.get('verify_certs')) - rule.setdefault('ca_certs', conf.get('ca_certs')) - rule.setdefault('client_cert', conf.get('client_cert')) - rule.setdefault('client_key', conf.get('client_key')) - - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - - # Set OpsGenie options from global config - rule.setdefault('opsgenie_default_receipients', None) - rule.setdefault('opsgenie_default_teams', None) - - # Make sure we have required options - if required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(required_locals - frozenset(rule.keys())))) - - if 'include' in rule and type(rule['include']) != list: - raise EAException('include option must be a list') - - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) - - if isinstance(rule.get('aggregation_key'), list): - rule['compound_aggregation_key'] = rule['aggregation_key'] - rule['aggregation_key'] = ','.join(rule['aggregation_key']) - - if isinstance(rule.get('compare_key'), list): - rule['compound_compare_key'] = rule['compare_key'] - rule['compare_key'] = ','.join(rule['compare_key']) - elif 'compare_key' in rule: - rule['compound_compare_key'] = [rule['compare_key']] - # Add QK, CK and timestamp to include - include = rule.get('include', ['*']) - if 'query_key' in rule: - include.append(rule['query_key']) - if 'compound_query_key' in rule: - include += rule['compound_query_key'] - if 'compound_aggregation_key' in rule: - include += rule['compound_aggregation_key'] - if 'compare_key' in rule: - include.append(rule['compare_key']) - if 'compound_compare_key' in rule: - include += rule['compound_compare_key'] - if 'top_count_keys' in rule: - include += rule['top_count_keys'] - include.append(rule['timestamp_field']) - rule['include'] = list(set(include)) - - # Check that generate_kibana_url is compatible with the filters - if rule.get('generate_kibana_link'): - for es_filter in rule.get('filter'): - if es_filter: - if 'not' in es_filter: - es_filter = es_filter['not'] - if 'query' in es_filter: - es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): - raise EAException('generate_kibana_link is incompatible with filters other than term, query_string and range. ' - 'Consider creating a dashboard and using use_kibana_dashboard instead.') - # Check that doc_type is provided if use_count/terms_query - if rule.get('use_count_query') or rule.get('use_terms_query'): - if 'doc_type' not in rule: - raise EAException('doc_type must be specified.') - - # Check that query_key is set if use_terms_query - if rule.get('use_terms_query'): - if 'query_key' not in rule: - raise EAException('query_key must be specified with use_terms_query') - - # Warn if use_strf_index is used with %y, %M or %D - # (%y = short year, %M = minutes, %D = full date) - if rule.get('use_strftime_index'): - for token in ['%y', '%M', '%D']: - if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime(rule.get('index')))) - - if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): - raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') - - -def load_modules(rule, args=None): - """ Loads things that could be modules. Enhancements, alerts and rule type. """ - # Set match enhancements - match_enhancements = [] - for enhancement_name in rule.get('match_enhancements', []): - if enhancement_name in dir(enhancements): - enhancement = getattr(enhancements, enhancement_name) - else: - enhancement = get_module(enhancement_name) - if not issubclass(enhancement, enhancements.BaseEnhancement): - raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % (enhancement_name)) - match_enhancements.append(enhancement(rule)) - rule['match_enhancements'] = match_enhancements - - # Convert rule type into RuleType object - if rule['type'] in rules_mapping: - rule['type'] = rules_mapping[rule['type']] - else: - rule['type'] = get_module(rule['type']) - if not issubclass(rule['type'], ruletypes.RuleType): - raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) - - # Make sure we have required alert and type options - reqs = rule['type'].required_options - - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) - # Instantiate rule - try: - rule['type'] = rule['type'](rule, args) - except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] - # Instantiate alerts only if we're not in debug mode - # In debug mode alerts are not actually sent so don't bother instantiating them - if not args or not args.debug: - rule['alert'] = load_alerts(rule, alert_field=rule['alert']) - - -def isyaml(filename): - return filename.endswith('.yaml') or filename.endswith('.yml') - - -def get_file_paths(conf, use_rule=None): - # Passing a filename directly can bypass rules_folder and .yaml checks - if use_rule and os.path.isfile(use_rule): - return [use_rule] - rule_folder = conf['rules_folder'] - rule_files = [] - if conf['scan_subdirectories']: - for root, folders, files in os.walk(rule_folder): - for filename in files: - if use_rule and use_rule != filename: - continue - if isyaml(filename): - rule_files.append(os.path.join(root, filename)) - else: - for filename in os.listdir(rule_folder): - fullpath = os.path.join(rule_folder, filename) - if os.path.isfile(fullpath) and isyaml(filename): - rule_files.append(fullpath) - return rule_files - - -def load_alerts(rule, alert_field): - def normalize_config(alert): - """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. - This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): - return alert, rule - elif isinstance(alert, dict): - name, config = iter(alert.items()).next() - config_copy = copy.copy(rule) - config_copy.update(config) # warning, this (intentionally) mutates the rule dict - return name, config_copy - else: - raise EAException() - - def create_alert(alert, alert_config): - alert_class = alerts_mapping.get(alert) or get_module(alert) - if not issubclass(alert_class, alerts.Alerter): - raise EAException('Alert module %s is not a subclass of Alerter' % (alert)) - missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset(alert_config or []) - if missing_options: - raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) - return alert_class(alert_config) - - try: - if type(alert_field) != list: - alert_field = [alert_field] - - alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): alerts_order.get(a, 1)) - # Convert all alerts into Alerter objects - alert_field = [create_alert(a, b) for a, b in alert_field] - - except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] - - return alert_field - - -def load_rules(args): +def load_conf(args, defaults=None, overwrites=None): """ Creates a conf dictionary for ElastAlerter. Loads the global - config file and then each rule found in rules_folder. + config file and then each rule found in rules_folder. - :param args: The parsed arguments to ElastAlert - :return: The global configuration, a dictionary. - """ - names = [] + :param args: The parsed arguments to ElastAlert + :param defaults: Dictionary of default conf values + :param overwrites: Dictionary of conf values to override + :return: The global configuration, a dictionary. + """ filename = args.config - conf = yaml_loader(filename) - use_rule = args.rule + if filename: + conf = yaml_loader(filename) + else: + try: + conf = yaml_loader('config.yaml') + except FileNotFoundError: + raise EAException('No --config or config.yaml found') # init logging from config and set log levels according to command line options configure_logging(args, conf) - for env_var, conf_var in env_settings.items(): + for env_var, conf_var in list(env_settings.items()): val = env(env_var, None) if val is not None: conf[conf_var] = val + for key, value in (iter(defaults.items()) if defaults is not None else []): + if key not in conf: + conf[key] = value + + for key, value in (iter(overwrites.items()) if overwrites is not None else []): + conf[key] = value + # Make sure we have all required globals - if required_globals - frozenset(conf.keys()): - raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + if required_globals - frozenset(list(conf.keys())): + raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(list(conf.keys()))))) + conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) conf.setdefault('scroll_keepalive', '30s') conf.setdefault('max_scrolling_count', 0) conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) + conf.setdefault('rules_loader', 'file') # Convert run_every, buffer_time into a timedelta object try: @@ -492,33 +89,18 @@ def load_rules(args): else: conf['old_query_limit'] = datetime.timedelta(weeks=1) except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) + raise EAException('Invalid time format used: %s' % e) - global base_config - base_config = copy.deepcopy(conf) - - # Load each rule configuration file - rules = [] - rule_files = get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = load_configuration(rule_file, conf, args) - # A rule failed to load, don't try to process it - if (not rule): - logging.error('Invalid rule file skipped: %s' % rule_file) - continue - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) + # Initialise the rule loader and load each rule configuration + rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) + rules_loader = rules_loader_class(conf) + conf['rules_loader'] = rules_loader + # Make sure we have all the required globals for the loader + # Make sure we have all required globals + if rules_loader.required_globals - frozenset(list(conf.keys())): + raise EAException( + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(list(conf.keys()))))) - conf['rules'] = rules return conf @@ -552,32 +134,3 @@ def configure_logging(args, conf): tracer = logging.getLogger('elasticsearch.trace') tracer.setLevel(logging.INFO) tracer.addHandler(logging.FileHandler(args.es_debug_trace)) - - -def get_rule_hashes(conf, use_rule=None): - rule_files = get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = get_rulefile_hash(rule_file) - return rule_mod_times - - -def get_rulefile_hash(rule_file): - rulefile_hash = '' - if os.path.exists(rule_file): - with open(rule_file) as fh: - rulefile_hash = hashlib.sha1(fh.read()).digest() - for import_rule_file in import_rules.get(rule_file, []): - rulefile_hash += get_rulefile_hash(import_rule_file) - return rulefile_hash - - -def adjust_deprecated_values(rule): - # From rename of simple HTTP alerter - if rule.get('type') == 'simple': - rule['type'] = 'post' - if 'simple_proxy' in rule: - rule['http_post_proxy'] = rule['simple_proxy'] - if 'simple_webhook_url' in rule: - rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning('"simple" alerter has been renamed "post" and comptability may be removed in a future release.') diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 13aebeb66..a0858da70 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -1,22 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function - import argparse import getpass +import json import os import time -import json import elasticsearch.helpers import yaml -from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch.exceptions import NotFoundError from envparse import Env +from .auth import Auth + env = Env(ES_USE_SSL=bool) @@ -139,7 +138,7 @@ def is_atleastsix(es_version): def is_atleastsixtwo(es_version): - major, minor = map(int, es_version.split(".")[:2]) + major, minor = list(map(int, es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) @@ -161,6 +160,7 @@ def main(): parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') @@ -185,6 +185,8 @@ def main(): if os.path.isfile(args.config): filename = args.config + elif os.path.isfile('../config.yaml'): + filename = '../config.yaml' else: filename = '' @@ -204,34 +206,38 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None password = args.password if args.password else None aws_region = args.aws_region - host = args.host if args.host else raw_input('Enter Elasticsearch host: ') - port = args.port if args.port else int(raw_input('Enter Elasticsearch port: ')) + host = args.host if args.host else input('Enter Elasticsearch host: ') + port = args.port if args.port else int(input('Enter Elasticsearch port: ')) use_ssl = (args.ssl if args.ssl is not None - else raw_input('Use SSL? t/f: ').lower() in ('t', 'true')) + else input('Use SSL? t/f: ').lower() in ('t', 'true')) if use_ssl: verify_certs = (args.verify_certs if args.verify_certs is not None - else raw_input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) + else input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) else: verify_certs = True if args.no_auth is None and username is None: - username = raw_input('Enter optional basic-auth username (or leave blank): ') + username = input('Enter optional basic-auth username (or leave blank): ') password = getpass.getpass('Enter optional basic-auth password (or leave blank): ') url_prefix = (args.url_prefix if args.url_prefix is not None - else raw_input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) + else input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) send_get_body_as = args.send_get_body_as ca_certs = None client_cert = None client_key = None - index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') + index = args.index if args.index is not None else input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None - else raw_input('Name of existing index to copy? (Default None) ')) + else input('Name of existing index to copy? (Default None) ')) timeout = args.timeout diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 11e037a6e..30ef4a2a7 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -5,8 +5,10 @@ import json import logging import os +import random import signal import sys +import threading import time import timeit import traceback @@ -16,39 +18,41 @@ from socket import error import dateutil.tz -import kibana -import yaml -from alerts import DebugAlerter -from config import get_rule_hashes -from config import load_configuration -from config import load_rules +import pytz +from apscheduler.schedulers.background import BackgroundScheduler from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException +from elasticsearch.exceptions import NotFoundError from elasticsearch.exceptions import TransportError -from enhancements import DropMatchException -from ruletypes import FlatlineRule -from util import add_raw_postfix -from util import cronite_datetime_to_timestamp -from util import dt_to_ts -from util import dt_to_unix -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import lookup_es_key -from util import parse_deadline -from util import parse_duration -from util import pretty_ts -from util import replace_dots_in_field_names -from util import seconds -from util import set_es_key -from util import should_scrolling_continue -from util import total_seconds -from util import ts_add -from util import ts_now -from util import ts_to_dt -from util import unix_to_dt + +from . import kibana +from .kibana_discover import generate_kibana_discover_url +from .alerts import DebugAlerter +from .config import load_conf +from .enhancements import DropMatchException +from .ruletypes import FlatlineRule +from .util import add_raw_postfix +from .util import cronite_datetime_to_timestamp +from .util import dt_to_ts +from .util import dt_to_unix +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import lookup_es_key +from .util import parse_deadline +from .util import parse_duration +from .util import pretty_ts +from .util import replace_dots_in_field_names +from .util import seconds +from .util import set_es_key +from .util import should_scrolling_continue +from .util import total_seconds +from .util import ts_add +from .util import ts_now +from .util import ts_to_dt +from .util import unix_to_dt class ElastAlerter(object): @@ -64,6 +68,8 @@ class ElastAlerter(object): should not be passed directly from a configuration file, but must be populated by config.py:load_rules instead. """ + thread_data = threading.local() + def parse_args(self, args): parser = argparse.ArgumentParser() parser.add_argument( @@ -102,15 +108,43 @@ def parse_args(self, args): self.args = parser.parse_args(args) def __init__(self, args): + self.es_clients = {} self.parse_args(args) self.debug = self.args.debug self.verbose = self.args.verbose - self.conf = load_rules(self.args) + if self.verbose and self.debug: + elastalert_logger.info( + "Note: --debug and --verbose flags are set. --debug takes precedent." + ) + + if self.verbose or self.debug: + elastalert_logger.setLevel(logging.INFO) + + if self.debug: + elastalert_logger.info( + """Note: In debug mode, alerts will be logged to console but NOT actually sent. + To send them but remain verbose, use --verbose instead.""" + ) + + if not self.args.es_debug: + logging.getLogger('elasticsearch').setLevel(logging.WARNING) + + if self.args.es_debug_trace: + tracer = logging.getLogger('elasticsearch.trace') + tracer.setLevel(logging.INFO) + tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) + + self.conf = load_conf(self.args) + self.rules_loader = self.conf['rules_loader'] + self.rules = self.rules_loader.load(self.conf, self.args) + + print(len(self.rules), 'rules loaded') + self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -119,17 +153,15 @@ def __init__(self, args): self.from_addr = self.conf.get('from_addr', 'ElastAlert') self.smtp_host = self.conf.get('smtp_host', 'localhost') self.max_aggregation = self.conf.get('max_aggregation', 10000) - self.alerts_sent = 0 - self.cumulative_hits = 0 - self.num_hits = 0 - self.num_dupes = 0 - self.current_es = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) + self.rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.scheduler = BackgroundScheduler() self.string_multi_field_name = self.conf.get('string_multi_field_name', False) self.add_metadata_alert = self.conf.get('add_metadata_alert', False) self.show_disabled_rules = self.conf.get('show_disabled_rules', True) @@ -140,7 +172,7 @@ def __init__(self, args): for rule in self.rules: if not self.init_rule(rule): remove.append(rule) - map(self.rules.remove, remove) + list(map(self.rules.remove, remove)) if self.args.silence: self.silence() @@ -252,12 +284,12 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - if self.current_es.is_atleastsixsix(): - res = self.current_es.search(index=index, size=1, body=query, - _source_includes=[timestamp_field], ignore_unavailable=True) + if self.thread_data.current_es.is_atleastsixsix(): + res = self.thread_data.current_es.search(index=index, size=1, body=query, + _source_includes=[timestamp_field], ignore_unavailable=True) else: - res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], - ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], + ignore_unavailable=True) except ElasticsearchException as e: self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query}) return '1969-12-30T00:00:00Z' @@ -280,7 +312,7 @@ def process_hits(rule, hits): for hit in hits: # Merge fields and _source hit.setdefault('_source', {}) - for key, value in hit.get('fields', {}).items(): + for key, value in list(hit.get('fields', {}).items()): # Fields are returned as lists, assume any with length 1 are not arrays in _source # Except sometimes they aren't lists. This is dependent on ES version hit['_source'].setdefault(key, value[0] if type(value) is list and len(value) == 1 else value) @@ -302,11 +334,11 @@ def process_hits(rule, hits): if rule.get('compound_query_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_query_key']] - hit['_source'][rule['query_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['query_key']] = ', '.join([str(value) for value in values]) if rule.get('compound_aggregation_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_aggregation_key']] - hit['_source'][rule['aggregation_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['aggregation_key']] = ', '.join([str(value) for value in values]) processed_hits.append(hit['_source']) @@ -328,7 +360,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): to_ts_func=rule['dt_to_ts'], five=rule['five'], ) - if self.current_es.is_atleastsixsix(): + if self.thread_data.current_es.is_atleastsixsix(): extra_args = {'_source_includes': rule['include']} else: extra_args = {'_source_include': rule['include']} @@ -342,9 +374,9 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): try: if scroll: - res = self.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) + res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) else: - res = self.current_es.search( + res = self.thread_data.current_es.search( scroll=scroll_keepalive, index=index, size=rule.get('max_query_size', self.max_query_size), @@ -355,10 +387,10 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): if '_scroll_id' in res: rule['scroll_id'] = res['_scroll_id'] - if self.current_es.is_atleastseven(): - self.total_hits = int(res['hits']['total']['value']) + if self.thread_data.current_es.is_atleastseven(): + self.thread_data.total_hits = int(res['hits']['total']['value']) else: - self.total_hits = int(res['hits']['total']) + self.thread_data.total_hits = int(res['hits']['total']) if len(res.get('_shards', {}).get('failures', [])) > 0: try: @@ -378,16 +410,16 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): self.handle_error('Error running query: %s' % (e), {'rule': rule['name'], 'query': query}) return None hits = res['hits']['hits'] - self.num_hits += len(hits) + self.thread_data.num_hits += len(hits) lt = rule.get('use_local_time') status_log = "Queried rule %s from %s to %s: %s / %s hits" % ( rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), - self.num_hits, + self.thread_data.num_hits, len(hits) ) - if self.total_hits > rule.get('max_query_size', self.max_query_size): + if self.thread_data.total_hits > rule.get('max_query_size', self.max_query_size): elastalert_logger.info("%s (scrolling..)" % status_log) else: elastalert_logger.info(status_log) @@ -420,7 +452,7 @@ def get_hits_count(self, rule, starttime, endtime, index): ) try: - res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) + res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -429,7 +461,7 @@ def get_hits_count(self, rule, starttime, endtime, index): self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query}) return None - self.num_hits += res['count'] + self.thread_data.num_hits += res['count'] lt = rule.get('use_local_time') elastalert_logger.info( "Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']) @@ -475,7 +507,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non try: if not rule['five']: - res = self.current_es.deprecated_search( + res = self.thread_data.current_es.deprecated_search( index=index, doc_type=rule['doc_type'], body=query, @@ -483,8 +515,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.deprecated_search(index=index, doc_type=rule['doc_type'], - body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'], + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -499,7 +531,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non buckets = res['aggregations']['filtered']['counts']['buckets'] else: buckets = res['aggregations']['counts']['buckets'] - self.num_hits += len(buckets) + self.thread_data.num_hits += len(buckets) lt = rule.get('use_local_time') elastalert_logger.info( 'Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)) @@ -522,7 +554,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field']) try: if not rule['five']: - res = self.current_es.deprecated_search( + res = self.thread_data.current_es.deprecated_search( index=index, doc_type=rule.get('doc_type'), body=query, @@ -530,8 +562,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'), - body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'), + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -544,10 +576,10 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ else: payload = res['aggregations'] - if self.current_es.is_atleastseven(): - self.num_hits += res['hits']['total']['value'] + if self.thread_data.current_es.is_atleastseven(): + self.thread_data.num_hits += res['hits']['total']['value'] else: - self.num_hits += res['hits']['total'] + self.thread_data.num_hits += res['hits']['total'] return {endtime: payload} @@ -570,10 +602,10 @@ def remove_old_events(self, rule): buffer_time = rule.get('buffer_time', self.buffer_time) if rule.get('query_delay'): buffer_time += rule['query_delay'] - for _id, timestamp in rule['processed_hits'].iteritems(): + for _id, timestamp in rule['processed_hits'].items(): if now - timestamp > buffer_time: remove.append(_id) - map(rule['processed_hits'].pop, remove) + list(map(rule['processed_hits'].pop, remove)) def run_query(self, rule, start=None, end=None, scroll=False): """ Query for the rule and pass all of the results to the RuleType instance. @@ -603,7 +635,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if data: old_len = len(data) data = self.remove_duplicate_events(data, rule) - self.num_dupes += old_len - len(data) + self.thread_data.num_dupes += old_len - len(data) # There was an exception while querying if data is None: @@ -619,7 +651,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): rule_inst.add_data(data) try: - if rule.get('scroll_id') and self.num_hits < self.total_hits and should_scrolling_continue(rule): + if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits and should_scrolling_continue(rule): self.run_query(rule, start, end, scroll=True) except RuntimeError: # It's possible to scroll far enough to hit max recursive depth @@ -627,7 +659,10 @@ def run_query(self, rule, start=None, end=None, scroll=False): if 'scroll_id' in rule: scroll_id = rule.pop('scroll_id') - self.current_es.clear_scroll(scroll_id=scroll_id) + try: + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + except NotFoundError: + pass return True @@ -749,7 +784,7 @@ def get_query_key_value(self, rule, match): # get the value for the match's query_key (or none) to form the key used for the silence_cache. # Flatline ruletype sets "key" instead of the actual query_key if isinstance(rule['type'], FlatlineRule) and 'key' in match: - return unicode(match['key']) + return str(match['key']) return self.get_named_key_value(rule, match, 'query_key') def get_aggregation_key_value(self, rule, match): @@ -764,7 +799,7 @@ def get_named_key_value(self, rule, match, key_name): if key_value is not None: # Only do the unicode conversion if we actually found something) # otherwise we might transform None --> 'None' - key_value = unicode(key_value) + key_value = str(key_value) except KeyError: # Some matches may not have the specified key # use a special token for these @@ -819,7 +854,7 @@ def run_rule(self, rule, endtime, starttime=None): :return: The number of matches that the rule produced. """ run_start = time.time() - self.current_es = elasticsearch_client(rule) + self.thread_data.current_es = self.es_clients.setdefault(rule['name'], elasticsearch_client(rule)) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -841,9 +876,9 @@ def run_rule(self, rule, endtime, starttime=None): return 0 # Run the rule. If querying over a large time period, split it up into segments - self.num_hits = 0 - self.num_dupes = 0 - self.cumulative_hits = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] @@ -852,15 +887,15 @@ def run_rule(self, rule, endtime, starttime=None): tmp_endtime = tmp_endtime + segment_size if not self.run_query(rule, rule['starttime'], tmp_endtime): return 0 - self.cumulative_hits += self.num_hits - self.num_hits = 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits + self.thread_data.num_hits = 0 rule['starttime'] = tmp_endtime rule['type'].garbage_collect(tmp_endtime) if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: self.run_query(rule, tmp_endtime, endtime) - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] return 0 @@ -869,14 +904,14 @@ def run_rule(self, rule, endtime, starttime=None): else: if not self.run_query(rule, rule['starttime'], endtime): return 0 - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits rule['type'].garbage_collect(endtime) # Process any new matches num_matches = len(rule['type'].matches) while rule['type'].matches: match = rule['type'].matches.pop(0) - match['num_hits'] = self.cumulative_hits + match['num_hits'] = self.thread_data.cumulative_hits match['num_matches'] = num_matches # If realert is set, silence the rule for that duration @@ -922,7 +957,7 @@ def run_rule(self, rule, endtime, starttime=None): 'endtime': endtime, 'starttime': rule['original_starttime'], 'matches': num_matches, - 'hits': max(self.num_hits, self.cumulative_hits), + 'hits': max(self.thread_data.num_hits, self.thread_data.cumulative_hits), '@timestamp': ts_now(), 'time_taken': time_taken} self.writeback('elastalert_status', body) @@ -931,6 +966,9 @@ def run_rule(self, rule, endtime, starttime=None): def init_rule(self, new_rule, new=True): ''' Copies some necessary non-config state from an exiting rule to a new rule. ''' + if not new: + self.scheduler.remove_job(job_id=new_rule['name']) + try: self.modify_rule_for_ES5(new_rule) except TransportError as e: @@ -965,7 +1003,9 @@ def init_rule(self, new_rule, new=True): blank_rule = {'agg_matches': [], 'aggregate_alert_time': {}, 'current_aggregate_id': {}, - 'processed_hits': {}} + 'processed_hits': {}, + 'run_every': self.run_every, + 'has_run_once': False} rule = blank_rule # Set rule to either a blank template or existing rule with same name @@ -981,12 +1021,22 @@ def init_rule(self, new_rule, new=True): 'aggregate_alert_time', 'processed_hits', 'starttime', - 'minimum_starttime'] + 'minimum_starttime', + 'has_run_once', + 'run_every'] for prop in copy_properties: if prop not in rule: continue new_rule[prop] = rule[prop] + job = self.scheduler.add_job(self.handle_rule_execution, 'interval', + args=[new_rule], + seconds=new_rule['run_every'].total_seconds(), + id=new_rule['name'], + max_instances=1, + jitter=5) + job.modify(next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=random.randint(0, 15))) + return new_rule @staticmethod @@ -1009,22 +1059,28 @@ def modify_rule_for_ES5(new_rule): new_rule['filter'] = new_filters def load_rule_changes(self): - ''' Using the modification times of rule config files, syncs the running rules - to match the files in rules_folder by removing, adding or reloading rules. ''' - new_rule_hashes = get_rule_hashes(self.conf, self.args.rule) + """ Using the modification times of rule config files, syncs the running rules + to match the files in rules_folder by removing, adding or reloading rules. """ + new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes - for rule_file, hash_value in self.rule_hashes.iteritems(): + for rule_file, hash_value in self.rule_hashes.items(): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) - self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file] + for rule in self.rules: + if rule['rule_file'] == rule_file: + break + else: + continue + self.scheduler.remove_job(job_id=rule['name']) + self.rules.remove(rule) continue if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = load_configuration(rule_file, self.conf) - if (not new_rule): + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) + if not new_rule: logging.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: @@ -1036,12 +1092,11 @@ def load_rule_changes(self): message = 'Could not load rule %s: %s' % (rule_file, e) self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. - with open(rule_file) as f: - try: - rule_yaml = yaml.load(f, Loader=yaml.FullLoader) - except yaml.scanner.ScannerError: - self.send_notification_email(exception=e) - continue + try: + rule_yaml = self.rules_loader.load_yaml(rule_file) + except EAException: + self.send_notification_email(exception=e) + continue self.send_notification_email(exception=e, rule=rule_yaml) continue @@ -1064,8 +1119,8 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = load_configuration(rule_file, self.conf) - if (not new_rule): + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) + if not new_rule: logging.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: @@ -1078,6 +1133,8 @@ def load_rule_changes(self): continue if self.init_rule(new_rule): elastalert_logger.info('Loaded new rule %s' % (rule_file)) + if new_rule['name'] in self.es_clients: + self.es_clients.pop(new_rule['name']) self.rules.append(new_rule) self.rule_hashes = new_rule_hashes @@ -1093,14 +1150,20 @@ def start(self): except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (self.starttime)) exit(1) + + for rule in self.rules: + rule['initial_starttime'] = self.starttime self.wait_until_responsive(timeout=self.args.timeout) self.running = True elastalert_logger.info("Starting up") + self.scheduler.add_job(self.handle_pending_alerts, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_pending_alerts') + self.scheduler.add_job(self.handle_config_change, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_config_change') + self.scheduler.start() while self.running: next_run = datetime.datetime.utcnow() + self.run_every - self.run_all_rules() - # Quit after end_time has been reached if self.args.end: endtime = ts_to_dt(self.args.end) @@ -1134,7 +1197,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1142,8 +1205,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1155,53 +1218,95 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): def run_all_rules(self): """ Run each rule one time """ + self.handle_pending_alerts() + + for rule in self.rules: + self.handle_rule_execution(rule) + + self.handle_config_change() + + def handle_pending_alerts(self): + self.thread_data.alerts_sent = 0 self.send_pending_alerts() + elastalert_logger.info("Background alerts thread %s pending alerts sent at %s" % (self.thread_data.alerts_sent, + pretty_ts(ts_now()))) - next_run = datetime.datetime.utcnow() + self.run_every + def handle_config_change(self): + if not self.args.pin_rules: + self.load_rule_changes() + elastalert_logger.info("Background configuration change check run at %s" % (pretty_ts(ts_now()))) + + def handle_rule_execution(self, rule): + self.thread_data.alerts_sent = 0 + next_run = datetime.datetime.utcnow() + rule['run_every'] + # Set endtime based on the rule's delay + delay = rule.get('query_delay') + if hasattr(self.args, 'end') and self.args.end: + endtime = ts_to_dt(self.args.end) + elif delay: + endtime = ts_now() - delay + else: + endtime = ts_now() + + # Apply rules based on execution time limits + if rule.get('limit_execution'): + rule['next_starttime'] = None + rule['next_min_starttime'] = None + exec_next = next(croniter(rule['limit_execution'])) + endtime_epoch = dt_to_unix(endtime) + # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time + # That means that we need to pause execution after this run + if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59: + # apscheduler requires pytz tzinfos, so don't use unix_to_dt here! + rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc) + if rule.get('limit_execution_coverage'): + rule['next_min_starttime'] = rule['next_starttime'] + if not rule['has_run_once']: + self.reset_rule_schedule(rule) + return - for rule in self.rules: - # Set endtime based on the rule's delay - delay = rule.get('query_delay') - if hasattr(self.args, 'end') and self.args.end: - endtime = ts_to_dt(self.args.end) - elif delay: - endtime = ts_now() - delay - else: - endtime = ts_now() + rule['has_run_once'] = True + try: + num_matches = self.run_rule(rule, endtime, rule.get('initial_starttime')) + except EAException as e: + self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) + except Exception as e: + self.handle_uncaught_exception(e, rule) + else: + old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) + elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," + " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), + self.thread_data.num_hits, self.thread_data.num_dupes, num_matches, + self.thread_data.alerts_sent)) + self.thread_data.alerts_sent = 0 - try: - num_matches = self.run_rule(rule, endtime, self.starttime) - except EAException as e: - self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) - except Exception as e: - self.handle_uncaught_exception(e, rule) - else: - old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) - total_hits = max(self.num_hits, self.cumulative_hits) - elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," - " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), - total_hits, self.num_dupes, num_matches, self.alerts_sent)) - self.alerts_sent = 0 - - if next_run < datetime.datetime.utcnow(): - # We were processing for longer than our refresh interval - # This can happen if --start was specified with a large time period - # or if we are running too slow to process events in real time. - logging.warning( - "Querying from %s to %s took longer than %s!" % ( - old_starttime, - pretty_ts(endtime, rule.get('use_local_time')), - self.run_every - ) + if next_run < datetime.datetime.utcnow(): + # We were processing for longer than our refresh interval + # This can happen if --start was specified with a large time period + # or if we are running too slow to process events in real time. + logging.warning( + "Querying from %s to %s took longer than %s!" % ( + old_starttime, + pretty_ts(endtime, rule.get('use_local_time')), + self.run_every ) + ) - self.remove_old_events(rule) + rule['initial_starttime'] = None - # Only force starttime once - self.starttime = None + self.remove_old_events(rule) - if not self.args.pin_rules: - self.load_rule_changes() + self.reset_rule_schedule(rule) + + def reset_rule_schedule(self, rule): + # We hit the end of a execution schedule, pause ourselves until next run + if rule.get('limit_execution') and rule['next_starttime']: + self.scheduler.modify_job(job_id=rule['name'], next_run_time=rule['next_starttime']) + # If we are preventing covering non-scheduled time periods, reset min_starttime and previous_endtime + if rule['next_min_starttime']: + rule['minimum_starttime'] = rule['next_min_starttime'] + rule['previous_endtime'] = rule['next_min_starttime'] + elastalert_logger.info('Pausing %s until next run at %s' % (rule['name'], pretty_ts(rule['next_starttime']))) def stop(self): """ Stop an ElastAlert runner that's been started """ @@ -1307,7 +1412,7 @@ def get_dashboard(self, rule, db_name): # TODO use doc_type = _doc res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: - raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] + raise EAException("Error querying for dashboard: %s" % (e)).with_traceback(sys.exc_info()[2]) if res['hits']['hits']: return json.loads(res['hits']['hits'][0]['_source']['dashboard']) @@ -1399,6 +1504,11 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link + if rule.get('generate_kibana_discover_url'): + kb_link = generate_kibana_discover_url(rule, matches[0]) + if kb_link: + matches[0]['kibana_discover_url'] = kb_link + # Enhancements were already run at match time if # run_enhancements_first is set or # retried==True, which means this is a retry of a failed alert @@ -1409,7 +1519,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): try: enhancement.process(match) valid_matches.append(match) - except DropMatchException as e: + except DropMatchException: pass except EAException as e: self.handle_error("Error running match enhancement: %s" % (e), {'rule': rule['name']}) @@ -1437,7 +1547,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): self.handle_error('Error while running alert %s: %s' % (alert.get_info()['type'], e), {'rule': rule['name']}) alert_exception = str(e) else: - self.alerts_sent += 1 + self.thread_data.alerts_sent += 1 alert_sent = True # Write the alert(s) to ES @@ -1447,7 +1557,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1460,6 +1570,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No 'alert_time': alert_time } + if rule.get('include_match_in_root'): + body.update({k: v for k, v in match.items() if not k.startswith('_')}) + if self.add_metadata_alert: body['category'] = rule['category'] body['description'] = rule['description'] @@ -1477,14 +1590,14 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): + def writeback(self, doc_type, body, rule=None, match_body=None): # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: writeback_body = replace_dots_in_field_names(body) else: writeback_body = body - for key in writeback_body.keys(): + for key in list(writeback_body.keys()): # Convert any datetime objects to timestamps if isinstance(writeback_body[key], datetime.datetime): writeback_body[key] = dt_to_ts(writeback_body[key]) @@ -1557,7 +1670,7 @@ def send_pending_alerts(self): continue # Set current_es for top_count_keys query - self.current_es = elasticsearch_client(rule) + self.thread_data.current_es = elasticsearch_client(rule) # Send the alert unless it's a future alert if ts_now() > ts_to_dt(alert_time): @@ -1573,7 +1686,7 @@ def send_pending_alerts(self): self.alert([match_body], rule, alert_time=alert_time, retried=retried) if rule['current_aggregate_id']: - for qk, agg_id in rule['current_aggregate_id'].iteritems(): + for qk, agg_id in rule['current_aggregate_id'].items(): if agg_id == _id: rule['current_aggregate_id'].pop(qk) break @@ -1590,7 +1703,7 @@ def send_pending_alerts(self): # Send in memory aggregated alerts for rule in self.rules: if rule['agg_matches']: - for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].iteritems(): + for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].items(): if ts_now() > aggregate_alert_time: alertable_matches = [ agg_match @@ -1717,7 +1830,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1801,7 +1914,7 @@ def is_silenced(self, rule_name): if res['hits']['hits']: until_ts = res['hits']['hits'][0]['_source']['until'] exponent = res['hits']['hits'][0]['_source'].get('exponent', 0) - if rule_name not in self.silence_cache.keys(): + if rule_name not in list(self.silence_cache.keys()): self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent) else: self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1]) @@ -1826,6 +1939,7 @@ def handle_uncaught_exception(self, exception, rule): if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] self.disabled_rules.append(rule) + self.scheduler.pause_job(job_id=rule['name']) elastalert_logger.info('Rule %s disabled', rule['name']) if self.notify_email: self.send_notification_email(exception=exception, rule=rule) @@ -1848,13 +1962,13 @@ def send_notification_email(self, text='', exception=None, rule=None, subject=No tb = traceback.format_exc() email_body += tb - if isinstance(self.notify_email, basestring): + if isinstance(self.notify_email, str): self.notify_email = [self.notify_email] email = MIMEText(email_body) email['Subject'] = subject if subject else 'ElastAlert notification' recipients = self.notify_email if rule and rule.get('notify_email'): - if isinstance(rule['notify_email'], basestring): + if isinstance(rule['notify_email'], str): rule['notify_email'] = [rule['notify_email']] recipients = recipients + rule['notify_email'] recipients = list(set(recipients)) @@ -1881,14 +1995,14 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): if hits_terms is None: top_events_count = {} else: - buckets = hits_terms.values()[0] + buckets = list(hits_terms.values())[0] # get_hits_terms adds to num_hits, but we don't want to count these - self.num_hits -= len(buckets) + self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] - counts = terms.items() + counts = list(terms.items()) counts.sort(key=lambda x: x[1], reverse=True) top_events_count = dict(counts[:number]) diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index d6c902514..6cc1cdd57 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from .util import pretty_ts class BaseEnhancement(object): @@ -14,6 +15,11 @@ def process(self, match): raise NotImplementedError() +class TimeEnhancement(BaseEnhancement): + def process(self, match): + match['@timestamp'] = pretty_ts(match['@timestamp']) + + class DropMatchException(Exception): """ ElastAlert will drop a match if this exception type is raised by an enhancement """ pass diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index ecc30c8ed..645a67762 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -1,4 +1,17 @@ { + "numeric_detection": true, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "rule_name": { "type": "keyword" @@ -16,8 +29,7 @@ "format": "dateOptionalTime" }, "match_body": { - "type": "object", - "enabled": "false" + "type": "object" }, "aggregate_id": { "type": "keyword" diff --git a/elastalert/kibana.py b/elastalert/kibana.py index 2cd557bff..de690494e 100644 --- a/elastalert/kibana.py +++ b/elastalert/kibana.py @@ -1,173 +1,176 @@ # -*- coding: utf-8 -*- +# flake8: noqa import os.path -import urllib +import urllib.error +import urllib.parse +import urllib.request -from util import EAException +from .util import EAException dashboard_temp = {'editable': True, - u'failover': False, - u'index': {u'default': u'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', - u'interval': u'none', - u'pattern': u'', - u'warm_fields': True}, - u'loader': {u'hide': False, - u'load_elasticsearch': True, - u'load_elasticsearch_size': 20, - u'load_gist': True, - u'load_local': True, - u'save_default': True, - u'save_elasticsearch': True, - u'save_gist': False, - u'save_local': True, - u'save_temp': True, - u'save_temp_ttl': u'30d', - u'save_temp_ttl_enable': True}, - u'nav': [{u'collapse': False, - u'enable': True, - u'filter_id': 0, - u'notice': False, - u'now': False, - u'refresh_intervals': [u'5s', - u'10s', - u'30s', - u'1m', - u'5m', - u'15m', - u'30m', - u'1h', - u'2h', - u'1d'], - u'status': u'Stable', - u'time_options': [u'5m', - u'15m', - u'1h', - u'6h', - u'12h', - u'24h', - u'2d', - u'7d', - u'30d'], - u'timefield': u'@timestamp', - u'type': u'timepicker'}], - u'panel_hints': True, - u'pulldowns': [{u'collapse': False, - u'enable': True, - u'notice': True, - u'type': u'filtering'}], - u'refresh': False, - u'rows': [{u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'annotate': {u'enable': False, - u'field': u'_type', - u'query': u'*', - u'size': 20, - u'sort': [u'_score', u'desc']}, - u'auto_int': True, - u'bars': True, - u'derivative': False, - u'editable': True, - u'fill': 3, - u'grid': {u'max': None, u'min': 0}, - u'group': [u'default'], - u'interactive': True, - u'interval': u'1m', - u'intervals': [u'auto', - u'1s', - u'1m', - u'5m', - u'10m', - u'30m', - u'1h', - u'3h', - u'12h', - u'1d', - u'1w', - u'1M', - u'1y'], - u'legend': True, - u'legend_counts': True, - u'lines': False, - u'linewidth': 3, - u'mode': u'count', - u'options': True, - u'percentage': False, - u'pointradius': 5, - u'points': False, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'resolution': 100, - u'scale': 1, - u'show_query': True, - u'span': 12, - u'spyable': True, - u'stack': True, - u'time_field': u'@timestamp', - u'timezone': u'browser', - u'title': u'Events over time', - u'tooltip': {u'query_as_alias': True, - u'value_type': u'cumulative'}, - u'type': u'histogram', - u'value_field': None, - u'x-axis': True, - u'y-axis': True, - u'y_format': u'none', - u'zerofill': True, - u'zoomlinks': True}], - u'title': u'Graph'}, - {u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'all_fields': False, - u'editable': True, - u'error': False, - u'field_list': True, - u'fields': [], - u'group': [u'default'], - u'header': True, - u'highlight': [], - u'localTime': True, - u'normTimes': True, - u'offset': 0, - u'overflow': u'min-height', - u'pages': 5, - u'paging': True, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'size': 100, - u'sort': [u'@timestamp', u'desc'], - u'sortable': True, - u'span': 12, - u'spyable': True, - u'status': u'Stable', - u'style': {u'font-size': u'9pt'}, - u'timeField': u'@timestamp', - u'title': u'All events', - u'trimFactor': 300, - u'type': u'table'}], - u'title': u'Events'}], - u'services': {u'filter': {u'ids': [0], - u'list': {u'0': {u'active': True, - u'alias': u'', - u'field': u'@timestamp', - u'from': u'now-24h', - u'id': 0, - u'mandate': u'must', - u'to': u'now', - u'type': u'time'}}}, - u'query': {u'ids': [0], - u'list': {u'0': {u'alias': u'', - u'color': u'#7EB26D', - u'enable': True, - u'id': 0, - u'pin': False, - u'query': u'', - u'type': u'lucene'}}}}, - u'style': u'dark', - u'title': u'ElastAlert Alert Dashboard'} + 'failover': False, + 'index': {'default': 'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', + 'interval': 'none', + 'pattern': '', + 'warm_fields': True}, + 'loader': {'hide': False, + 'load_elasticsearch': True, + 'load_elasticsearch_size': 20, + 'load_gist': True, + 'load_local': True, + 'save_default': True, + 'save_elasticsearch': True, + 'save_gist': False, + 'save_local': True, + 'save_temp': True, + 'save_temp_ttl': '30d', + 'save_temp_ttl_enable': True}, + 'nav': [{'collapse': False, + 'enable': True, + 'filter_id': 0, + 'notice': False, + 'now': False, + 'refresh_intervals': ['5s', + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d'], + 'status': 'Stable', + 'time_options': ['5m', + '15m', + '1h', + '6h', + '12h', + '24h', + '2d', + '7d', + '30d'], + 'timefield': '@timestamp', + 'type': 'timepicker'}], + 'panel_hints': True, + 'pulldowns': [{'collapse': False, + 'enable': True, + 'notice': True, + 'type': 'filtering'}], + 'refresh': False, + 'rows': [{'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'annotate': {'enable': False, + 'field': '_type', + 'query': '*', + 'size': 20, + 'sort': ['_score', 'desc']}, + 'auto_int': True, + 'bars': True, + 'derivative': False, + 'editable': True, + 'fill': 3, + 'grid': {'max': None, 'min': 0}, + 'group': ['default'], + 'interactive': True, + 'interval': '1m', + 'intervals': ['auto', + '1s', + '1m', + '5m', + '10m', + '30m', + '1h', + '3h', + '12h', + '1d', + '1w', + '1M', + '1y'], + 'legend': True, + 'legend_counts': True, + 'lines': False, + 'linewidth': 3, + 'mode': 'count', + 'options': True, + 'percentage': False, + 'pointradius': 5, + 'points': False, + 'queries': {'ids': [0], 'mode': 'all'}, + 'resolution': 100, + 'scale': 1, + 'show_query': True, + 'span': 12, + 'spyable': True, + 'stack': True, + 'time_field': '@timestamp', + 'timezone': 'browser', + 'title': 'Events over time', + 'tooltip': {'query_as_alias': True, + 'value_type': 'cumulative'}, + 'type': 'histogram', + 'value_field': None, + 'x-axis': True, + 'y-axis': True, + 'y_format': 'none', + 'zerofill': True, + 'zoomlinks': True}], + 'title': 'Graph'}, + {'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'all_fields': False, + 'editable': True, + 'error': False, + 'field_list': True, + 'fields': [], + 'group': ['default'], + 'header': True, + 'highlight': [], + 'localTime': True, + 'normTimes': True, + 'offset': 0, + 'overflow': 'min-height', + 'pages': 5, + 'paging': True, + 'queries': {'ids': [0], 'mode': 'all'}, + 'size': 100, + 'sort': ['@timestamp', 'desc'], + 'sortable': True, + 'span': 12, + 'spyable': True, + 'status': 'Stable', + 'style': {'font-size': '9pt'}, + 'timeField': '@timestamp', + 'title': 'All events', + 'trimFactor': 300, + 'type': 'table'}], + 'title': 'Events'}], + 'services': {'filter': {'ids': [0], + 'list': {'0': {'active': True, + 'alias': '', + 'field': '@timestamp', + 'from': 'now-24h', + 'id': 0, + 'mandate': 'must', + 'to': 'now', + 'type': 'time'}}}, + 'query': {'ids': [0], + 'list': {'0': {'alias': '', + 'color': '#7EB26D', + 'enable': True, + 'id': 0, + 'pin': False, + 'query': '', + 'type': 'lucene'}}}}, + 'style': 'dark', + 'title': 'ElastAlert Alert Dashboard'} kibana4_time_temp = "(refreshInterval:(display:Off,section:0,value:0),time:(from:'%s',mode:absolute,to:'%s'))" @@ -213,9 +216,9 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = es_filter['query_string']['query'] elif 'term' in es_filter: kibana_filter['type'] = 'field' - f_field, f_query = es_filter['term'].items()[0] + f_field, f_query = list(es_filter['term'].items())[0] # Wrap query in quotes, otherwise certain characters cause Kibana to throw errors - if isinstance(f_query, basestring): + if isinstance(f_query, str): f_query = '"%s"' % (f_query.replace('"', '\\"')) if isinstance(f_query, list): # Escape quotes @@ -228,7 +231,7 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = f_query elif 'range' in es_filter: kibana_filter['type'] = 'range' - f_field, f_range = es_filter['range'].items()[0] + f_field, f_range = list(es_filter['range'].items())[0] kibana_filter['field'] = f_field kibana_filter.update(f_range) else: @@ -250,7 +253,7 @@ def filters_from_dashboard(db): filters = db['services']['filter']['list'] config_filters = [] or_filters = [] - for filter in filters.values(): + for filter in list(filters.values()): filter_type = filter['type'] if filter_type == 'time': continue @@ -281,5 +284,5 @@ def filters_from_dashboard(db): def kibana4_dashboard_link(dashboard, starttime, endtime): dashboard = os.path.expandvars(dashboard) time_settings = kibana4_time_temp % (starttime, endtime) - time_settings = urllib.quote(time_settings) + time_settings = urllib.parse.quote(time_settings) return "%s?_g=%s" % (dashboard, time_settings) diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py new file mode 100644 index 000000000..7e4dbb5d1 --- /dev/null +++ b/elastalert/kibana_discover.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import lookup_es_key +from .util import ts_add + +kibana_default_timedelta = datetime.timedelta(minutes=10) + +kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) + +def generate_kibana_discover_url(rule, match): + ''' Creates a link for a kibana discover app. ''' + + discover_app_url = rule.get('kibana_discover_app_url') + if not discover_app_url: + logging.warning( + 'Missing kibana_discover_app_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: + logging.warning( + 'Missing kibana_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('kibana_discover_index_pattern_id') + if not index: + logging.warning( + 'Missing kibana_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('kibana_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', kibana_default_timedelta) + from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if kibana_version in kibana5_kibana6_versions: + globalState = kibana6_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + elif kibana_version in kibana7_versions: + globalState = kibana7_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + else: + logging.warning( + 'Unknown kibana discover application version %s for rule %s' % ( + kibana_version, + rule.get('name', '') + ) + ) + return None + + return "%s?_g=%s&_a=%s" % ( + os.path.expandvars(discover_app_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState) + ) + + +def kibana6_disover_global_state(from_time, to_time): + return prison.dumps( { + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'mode': 'absolute', + 'to': to_time + } + } ) + + +def kibana7_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def kibana_discover_app_state(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + bool_filter = { 'must': filters } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'bool': bool_filter, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'bool', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'columns': columns, + 'filters': app_filters, + 'index': index, + 'interval': 'auto' + } ) diff --git a/elastalert/loaders.py b/elastalert/loaders.py new file mode 100644 index 000000000..c602d0dc5 --- /dev/null +++ b/elastalert/loaders.py @@ -0,0 +1,552 @@ +# -*- coding: utf-8 -*- +import copy +import datetime +import hashlib +import logging +import os +import sys + +import jsonschema +import yaml +import yaml.scanner +from staticconf.loader import yaml_loader + +from . import alerts +from . import enhancements +from . import ruletypes +from .opsgenie import OpsGenieAlerter +from .util import dt_to_ts +from .util import dt_to_ts_with_format +from .util import dt_to_unix +from .util import dt_to_unixms +from .util import EAException +from .util import get_module +from .util import ts_to_dt +from .util import ts_to_dt_with_format +from .util import unix_to_dt +from .util import unixms_to_dt + + +class RulesLoader(object): + # import rule dependency + import_rules = {} + + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset([]) + + # Required local (rule.yaml) configuration options + required_locals = frozenset(['alert', 'type', 'name', 'index']) + + # Used to map the names of rules to their classes + rules_mapping = { + 'frequency': ruletypes.FrequencyRule, + 'any': ruletypes.AnyRule, + 'spike': ruletypes.SpikeRule, + 'blacklist': ruletypes.BlacklistRule, + 'whitelist': ruletypes.WhitelistRule, + 'change': ruletypes.ChangeRule, + 'flatline': ruletypes.FlatlineRule, + 'new_term': ruletypes.NewTermsRule, + 'cardinality': ruletypes.CardinalityRule, + 'metric_aggregation': ruletypes.MetricAggregationRule, + 'percentage_match': ruletypes.PercentageMatchRule, + } + + # Used to map names of alerts to their classes + alerts_mapping = { + 'email': alerts.EmailAlerter, + 'jira': alerts.JiraAlerter, + 'opsgenie': OpsGenieAlerter, + 'stomp': alerts.StompAlerter, + 'debug': alerts.DebugAlerter, + 'command': alerts.CommandAlerter, + 'sns': alerts.SnsAlerter, + 'hipchat': alerts.HipChatAlerter, + 'stride': alerts.StrideAlerter, + 'ms_teams': alerts.MsTeamsAlerter, + 'slack': alerts.SlackAlerter, + 'mattermost': alerts.MattermostAlerter, + 'pagerduty': alerts.PagerDutyAlerter, + 'exotel': alerts.ExotelAlerter, + 'twilio': alerts.TwilioAlerter, + 'victorops': alerts.VictorOpsAlerter, + 'telegram': alerts.TelegramAlerter, + 'googlechat': alerts.GoogleChatAlerter, + 'gitter': alerts.GitterAlerter, + 'servicenow': alerts.ServiceNowAlerter, + 'alerta': alerts.AlertaAlerter, + 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter + } + + # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list + # For example, jira goes before email so the ticket # will be added to the resulting email. + alerts_order = { + 'jira': 0, + 'email': 1 + } + + base_config = {} + + def __init__(self, conf): + # schema for rule yaml + self.rule_schema = jsonschema.Draft7Validator( + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) + + self.base_config = copy.deepcopy(conf) + + def load(self, conf, args=None): + """ + Discover and load all the rules as defined in the conf and args. + :param dict conf: Configuration dict + :param dict args: Arguments dict + :return: List of rules + :rtype: list + """ + names = [] + use_rule = None if args is None else args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.get_names(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # A rule failed to load, don't try to process it + if not rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_names(self, conf, use_rule=None): + """ + Return a list of rule names that can be passed to `get_yaml` to retrieve. + :param dict conf: Configuration dict + :param str use_rule: Limit to only specified rule + :return: A list of rule names + :rtype: list + """ + raise NotImplementedError() + + def get_hashes(self, conf, use_rule=None): + """ + Discover and get the hashes of all the rules as defined in the conf. + :param dict conf: Configuration + :param str use_rule: Limit to only specified rule + :return: Dict of rule name to hash + :rtype: dict + """ + raise NotImplementedError() + + def get_yaml(self, filename): + """ + Get and parse the yaml of the specified rule. + :param str filename: Rule to get the yaml + :return: Rule YAML dict + :rtype: dict + """ + raise NotImplementedError() + + def get_import_rule(self, rule): + """ + Retrieve the name of the rule to import. + :param dict rule: Rule dict + :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + :rtype: str + """ + return rule['import'] + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param str filename: The name of a rule configuration file. + :param dict conf: The global configuration dictionary, used for populating defaults. + :param dict args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule + + def load_yaml(self, filename): + """ + Load the rule including all dependency rules. + :param str filename: Rule to load + :return: Loaded rule dict + :rtype: dict + """ + rule = { + 'rule_file': filename, + } + + self.import_rules.pop(filename, None) # clear `filename` dependency + while True: + loaded = self.get_yaml(filename) + + # Special case for merging filters - if both files specify a filter merge (AND) them + if 'filter' in rule and 'filter' in loaded: + rule['filter'] = loaded['filter'] + rule['filter'] + + loaded.update(rule) + rule = loaded + if 'import' in rule: + # Find the path of the next file. + import_filename = self.get_import_rule(rule) + # set dependencies + rules = self.import_rules.get(filename, []) + rules.append(import_filename) + self.import_rules[filename] = rules + filename = import_filename + del (rule['import']) # or we could go on forever! + else: + break + + return rule + + def load_options(self, rule, conf, filename, args=None): + """ Converts time objects, sets defaults, and validates some settings. + + :param rule: A dictionary of parsed YAML from a rule config file. + :param conf: The global configuration dictionary, used for populating defaults. + :param filename: Name of the rule + :param args: Arguments + """ + self.adjust_deprecated_values(rule) + + try: + self.rule_schema.validate(rule) + except jsonschema.ValidationError as e: + raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) + + try: + # Set all time based parameters + if 'timeframe' in rule: + rule['timeframe'] = datetime.timedelta(**rule['timeframe']) + if 'realert' in rule: + rule['realert'] = datetime.timedelta(**rule['realert']) + else: + if 'aggregation' in rule: + rule['realert'] = datetime.timedelta(minutes=0) + else: + rule['realert'] = datetime.timedelta(minutes=1) + if 'aggregation' in rule and not rule['aggregation'].get('schedule'): + rule['aggregation'] = datetime.timedelta(**rule['aggregation']) + if 'query_delay' in rule: + rule['query_delay'] = datetime.timedelta(**rule['query_delay']) + if 'buffer_time' in rule: + rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) + if 'bucket_interval' in rule: + rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) + if 'exponential_realert' in rule: + rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) + if 'kibana4_start_timedelta' in rule: + rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) + if 'kibana4_end_timedelta' in rule: + rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + if 'kibana_discover_from_timedelta' in rule: + rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + if 'kibana_discover_to_timedelta' in rule: + rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) + except (KeyError, TypeError) as e: + raise EAException('Invalid time format used: %s' % e) + + # Set defaults, copy defaults from config.yaml + for key, val in list(self.base_config.items()): + rule.setdefault(key, val) + rule.setdefault('name', os.path.splitext(filename)[0]) + rule.setdefault('realert', datetime.timedelta(seconds=0)) + rule.setdefault('aggregation', datetime.timedelta(seconds=0)) + rule.setdefault('query_delay', datetime.timedelta(seconds=0)) + rule.setdefault('timestamp_field', '@timestamp') + rule.setdefault('filter', []) + rule.setdefault('timestamp_type', 'iso') + rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') + rule.setdefault('_source_enabled', True) + rule.setdefault('use_local_time', True) + rule.setdefault('description', "") + + # Set timestamp_type conversion function, used when generating queries and processing hits + rule['timestamp_type'] = rule['timestamp_type'].strip().lower() + if rule['timestamp_type'] == 'iso': + rule['ts_to_dt'] = ts_to_dt + rule['dt_to_ts'] = dt_to_ts + elif rule['timestamp_type'] == 'unix': + rule['ts_to_dt'] = unix_to_dt + rule['dt_to_ts'] = dt_to_unix + elif rule['timestamp_type'] == 'unix_ms': + rule['ts_to_dt'] = unixms_to_dt + rule['dt_to_ts'] = dt_to_unixms + elif rule['timestamp_type'] == 'custom': + def _ts_to_dt_with_format(ts): + return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) + + def _dt_to_ts_with_format(dt): + ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) + if 'timestamp_format_expr' in rule: + # eval expression passing 'ts' and 'dt' + return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) + else: + return ts + + rule['ts_to_dt'] = _ts_to_dt_with_format + rule['dt_to_ts'] = _dt_to_ts_with_format + else: + raise EAException('timestamp_type must be one of iso, unix, or unix_ms') + + # Add support for client ssl certificate auth + if 'verify_certs' in conf: + rule.setdefault('verify_certs', conf.get('verify_certs')) + rule.setdefault('ca_certs', conf.get('ca_certs')) + rule.setdefault('client_cert', conf.get('client_cert')) + rule.setdefault('client_key', conf.get('client_key')) + + # Set HipChat options from global config + rule.setdefault('hipchat_msg_color', 'red') + rule.setdefault('hipchat_domain', 'api.hipchat.com') + rule.setdefault('hipchat_notify', True) + rule.setdefault('hipchat_from', '') + rule.setdefault('hipchat_ignore_ssl_errors', False) + + # Make sure we have required options + if self.required_locals - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) + + if 'include' in rule and type(rule['include']) != list: + raise EAException('include option must be a list') + + raw_query_key = rule.get('query_key') + if isinstance(raw_query_key, list): + if len(raw_query_key) > 1: + rule['compound_query_key'] = raw_query_key + rule['query_key'] = ','.join(raw_query_key) + elif len(raw_query_key) == 1: + rule['query_key'] = raw_query_key[0] + else: + del(rule['query_key']) + + if isinstance(rule.get('aggregation_key'), list): + rule['compound_aggregation_key'] = rule['aggregation_key'] + rule['aggregation_key'] = ','.join(rule['aggregation_key']) + + if isinstance(rule.get('compare_key'), list): + rule['compound_compare_key'] = rule['compare_key'] + rule['compare_key'] = ','.join(rule['compare_key']) + elif 'compare_key' in rule: + rule['compound_compare_key'] = [rule['compare_key']] + # Add QK, CK and timestamp to include + include = rule.get('include', ['*']) + if 'query_key' in rule: + include.append(rule['query_key']) + if 'compound_query_key' in rule: + include += rule['compound_query_key'] + if 'compound_aggregation_key' in rule: + include += rule['compound_aggregation_key'] + if 'compare_key' in rule: + include.append(rule['compare_key']) + if 'compound_compare_key' in rule: + include += rule['compound_compare_key'] + if 'top_count_keys' in rule: + include += rule['top_count_keys'] + include.append(rule['timestamp_field']) + rule['include'] = list(set(include)) + + # Check that generate_kibana_url is compatible with the filters + if rule.get('generate_kibana_link'): + for es_filter in rule.get('filter'): + if es_filter: + if 'not' in es_filter: + es_filter = es_filter['not'] + if 'query' in es_filter: + es_filter = es_filter['query'] + if list(es_filter.keys())[0] not in ('term', 'query_string', 'range'): + raise EAException( + 'generate_kibana_link is incompatible with filters other than term, query_string and range.' + 'Consider creating a dashboard and using use_kibana_dashboard instead.') + + # Check that doc_type is provided if use_count/terms_query + if rule.get('use_count_query') or rule.get('use_terms_query'): + if 'doc_type' not in rule: + raise EAException('doc_type must be specified.') + + # Check that query_key is set if use_terms_query + if rule.get('use_terms_query'): + if 'query_key' not in rule: + raise EAException('query_key must be specified with use_terms_query') + + # Warn if use_strf_index is used with %y, %M or %D + # (%y = short year, %M = minutes, %D = full date) + if rule.get('use_strftime_index'): + for token in ['%y', '%M', '%D']: + if token in rule.get('index'): + logging.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) + + if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): + raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') + + def load_modules(self, rule, args=None): + """ Loads things that could be modules. Enhancements, alerts and rule type. """ + # Set match enhancements + match_enhancements = [] + for enhancement_name in rule.get('match_enhancements', []): + if enhancement_name in dir(enhancements): + enhancement = getattr(enhancements, enhancement_name) + else: + enhancement = get_module(enhancement_name) + if not issubclass(enhancement, enhancements.BaseEnhancement): + raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % enhancement_name) + match_enhancements.append(enhancement(rule)) + rule['match_enhancements'] = match_enhancements + + # Convert rule type into RuleType object + if rule['type'] in self.rules_mapping: + rule['type'] = self.rules_mapping[rule['type']] + else: + rule['type'] = get_module(rule['type']) + if not issubclass(rule['type'], ruletypes.RuleType): + raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) + + # Make sure we have required alert and type options + reqs = rule['type'].required_options + + if reqs - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(list(rule.keys()))))) + # Instantiate rule + try: + rule['type'] = rule['type'](rule, args) + except (KeyError, EAException) as e: + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)).with_traceback(sys.exc_info()[2]) + # Instantiate alerts only if we're not in debug mode + # In debug mode alerts are not actually sent so don't bother instantiating them + if not args or not args.debug: + rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) + + def load_alerts(self, rule, alert_field): + def normalize_config(alert): + """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. + This function normalizes them both to the latter format. """ + if isinstance(alert, str): + return alert, rule + elif isinstance(alert, dict): + name, config = next(iter(list(alert.items()))) + config_copy = copy.copy(rule) + config_copy.update(config) # warning, this (intentionally) mutates the rule dict + return name, config_copy + else: + raise EAException() + + def create_alert(alert, alert_config): + alert_class = self.alerts_mapping.get(alert) or get_module(alert) + if not issubclass(alert_class, alerts.Alerter): + raise EAException('Alert module %s is not a subclass of Alerter' % alert) + missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset( + alert_config or []) + if missing_options: + raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) + return alert_class(alert_config) + + try: + if type(alert_field) != list: + alert_field = [alert_field] + + alert_field = [normalize_config(x) for x in alert_field] + alert_field = sorted(alert_field, key=lambda a_b: self.alerts_order.get(a_b[0], 1)) + # Convert all alerts into Alerter objects + alert_field = [create_alert(a, b) for a, b in alert_field] + + except (KeyError, EAException) as e: + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)).with_traceback(sys.exc_info()[2]) + + return alert_field + + @staticmethod + def adjust_deprecated_values(rule): + # From rename of simple HTTP alerter + if rule.get('type') == 'simple': + rule['type'] = 'post' + if 'simple_proxy' in rule: + rule['http_post_proxy'] = rule['simple_proxy'] + if 'simple_webhook_url' in rule: + rule['http_post_url'] = rule['simple_webhook_url'] + logging.warning( + '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') + + +class FileRulesLoader(RulesLoader): + + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset(['rules_folder']) + + def get_names(self, conf, use_rule=None): + # Passing a filename directly can bypass rules_folder and .yaml checks + if use_rule and os.path.isfile(use_rule): + return [use_rule] + rule_folder = conf['rules_folder'] + rule_files = [] + if 'scan_subdirectories' in conf and conf['scan_subdirectories']: + for root, folders, files in os.walk(rule_folder): + for filename in files: + if use_rule and use_rule != filename: + continue + if self.is_yaml(filename): + rule_files.append(os.path.join(root, filename)) + else: + for filename in os.listdir(rule_folder): + fullpath = os.path.join(rule_folder, filename) + if os.path.isfile(fullpath) and self.is_yaml(filename): + rule_files.append(fullpath) + return rule_files + + def get_hashes(self, conf, use_rule=None): + rule_files = self.get_names(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + """ + Allow for relative paths to the import rule. + :param dict rule: + :return: Path the import rule + :rtype: str + """ + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def get_rule_file_hash(self, rule_file): + rule_file_hash = '' + if os.path.exists(rule_file): + with open(rule_file, 'rb') as fh: + rule_file_hash = hashlib.sha1(fh.read()).digest() + for import_rule_file in self.import_rules.get(rule_file, []): + rule_file_hash += self.get_rule_file_hash(import_rule_file) + return rule_file_hash + + @staticmethod + def is_yaml(filename): + return filename.endswith('.yaml') or filename.endswith('.yml') diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 30f5302bc..bcdaf2d05 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import json import logging +import os.path import requests -from alerts import Alerter -from alerts import BasicMatchString -from util import EAException -from util import elastalert_logger -from util import lookup_es_key + +from .alerts import Alerter +from .alerts import BasicMatchString +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key class OpsGenieAlerter(Alerter): @@ -31,15 +33,16 @@ def __init__(self, *args): self.alias = self.rule.get('opsgenie_alias') self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') + self.opsgenie_details = self.rule.get('opsgenie_details', {}) def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: formated_responders = list() - responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) - responders_values = dict((k, v) for k, v in responders_values.iteritems() if v) + responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.items()) + responders_values = dict((k, v) for k, v in responders_values.items() if v) for responder in responders: - responder = unicode(responder) + responder = str(responder) try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: @@ -60,7 +63,7 @@ def _fill_responders(self, responders, type_): def alert(self, matches): body = '' for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -95,6 +98,10 @@ def alert(self, matches): if self.alias is not None: post['alias'] = self.alias.format(**matches[0]) + details = self.get_details(matches) + if details: + post['details'] = details + logging.debug(json.dumps(post)) headers = { @@ -135,7 +142,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - opsgenie_subject = unicode(self.rule['opsgenie_subject']) + opsgenie_subject = str(self.rule['opsgenie_subject']) if self.opsgenie_subject_args: opsgenie_subject_values = [lookup_es_key(matches[0], arg) for arg in self.opsgenie_subject_args] @@ -160,3 +167,19 @@ def get_info(self): if self.teams: ret['teams'] = self.teams return ret + + def get_details(self, matches): + details = {} + + for key, value in self.opsgenie_details.items(): + + if type(value) is dict: + if 'field' in value: + field_value = lookup_es_key(matches[0], value['field']) + if field_value is not None: + details[key] = str(field_value) + + elif type(value) is str: + details[key] = os.path.expandvars(value) + + return details diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index ef1392b28..4a0634954 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import json import yaml @@ -12,10 +9,10 @@ def main(): - es_host = raw_input("Elasticsearch host: ") - es_port = raw_input("Elasticsearch port: ") - db_name = raw_input("Dashboard name: ") - send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' + es_host = input("Elasticsearch host: ") + es_port = input("Elasticsearch port: ") + db_name = input("Dashboard name: ") + send_get_body_as = input("Method for querying Elasticsearch[GET]: ") or 'GET' es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as}) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 5ffb867ed..86ae7c29f 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -4,19 +4,20 @@ import sys from blist import sortedlist -from util import add_raw_postfix -from util import dt_to_ts -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import hashable -from util import lookup_es_key -from util import new_get_event_ts -from util import pretty_ts -from util import total_seconds -from util import ts_now -from util import ts_to_dt + +from .util import add_raw_postfix +from .util import dt_to_ts +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import hashable +from .util import lookup_es_key +from .util import new_get_event_ts +from .util import pretty_ts +from .util import total_seconds +from .util import ts_now +from .util import ts_to_dt class RuleType(object): @@ -207,8 +208,8 @@ def add_match(self, match): if change: extra = {'old_value': change[0], 'new_value': change[1]} - elastalert_logger.debug("Description of the changed records " + str(dict(match.items() + extra.items()))) - super(ChangeRule, self).add_match(dict(match.items() + extra.items())) + elastalert_logger.debug("Description of the changed records " + str(dict(list(match.items()) + list(extra.items())))) + super(ChangeRule, self).add_match(dict(list(match.items()) + list(extra.items()))) class FrequencyRule(RuleType): @@ -226,14 +227,14 @@ def add_count_data(self, data): if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - (ts, count), = data.items() + (ts, count), = list(data.items()) event = ({self.ts_field: ts}, count) self.occurrences.setdefault('all', EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event) self.check_for_match('all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: event = ({self.ts_field: timestamp, self.rules['query_key']: bucket['key']}, bucket['doc_count']) @@ -276,10 +277,10 @@ def check_for_match(self, key, end=False): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ stale_keys = [] - for key, window in self.occurrences.iteritems(): + for key, window in self.occurrences.items(): if timestamp - lookup_es_key(window.data[-1][0], self.ts_field) > self.rules['timeframe']: stale_keys.append(key) - map(self.occurrences.pop, stale_keys) + list(map(self.occurrences.pop, stale_keys)) def get_match_str(self, match): lt = self.rules.get('use_local_time') @@ -403,11 +404,11 @@ def add_count_data(self, data): """ Add count data to the rule. Data should be of the form {ts: count}. """ if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - for ts, count in data.iteritems(): + for ts, count in data.items(): self.handle_event({self.ts_field: ts}, count, 'all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: count = bucket['doc_count'] event = {self.ts_field: timestamp, @@ -491,7 +492,7 @@ def add_match(self, match, qk): extra_info = {'spike_count': spike_count, 'reference_count': reference_count} - match = dict(match.items() + extra_info.items()) + match = dict(list(match.items()) + list(extra_info.items())) super(SpikeRule, self).add_match(match) @@ -537,7 +538,7 @@ def get_match_str(self, match): def garbage_collect(self, ts): # Windows are sized according to their newest event # This is a placeholder to accurately size windows in the absence of events - for qk in self.cur_windows.keys(): + for qk in list(self.cur_windows.keys()): # If we havn't seen this key in a long time, forget it if qk != 'all' and self.ref_windows[qk].count() == 0 and self.cur_windows[qk].count() == 0: self.cur_windows.pop(qk) @@ -610,7 +611,7 @@ def garbage_collect(self, ts): # We add an event with a count of zero to the EventWindow for each key. This will cause the EventWindow # to remove events that occurred more than one `timeframe` ago, and call onRemoved on them. default = ['all'] if 'query_key' not in self.rules else [] - for key in self.occurrences.keys() or default: + for key in list(self.occurrences.keys()) or default: self.occurrences.setdefault( key, EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts) @@ -652,7 +653,7 @@ def __init__(self, rule, args=None): self.get_all_terms(args) except Exception as e: # Refuse to start if we cannot get existing terms - raise EAException('Error searching for existing terms: %s' % (repr(e))), None, sys.exc_info()[2] + raise EAException('Error searching for existing terms: %s' % (repr(e))).with_traceback(sys.exc_info()[2]) def get_all_terms(self, args): """ Performs a terms aggregation for each field to get every existing term. """ @@ -733,7 +734,7 @@ def get_all_terms(self, args): time_filter[self.rules['timestamp_field']] = {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)} - for key, values in self.seen_values.iteritems(): + for key, values in self.seen_values.items(): if not values: if type(key) == tuple: # If we don't have any results, it could either be because of the absence of any baseline data @@ -881,7 +882,7 @@ def add_data(self, data): def add_terms_data(self, terms): # With terms query, len(self.fields) is always 1 and the 0'th entry is always a string field = self.fields[0] - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: if bucket['doc_count']: if bucket['key'] not in self.seen_values[field]: @@ -1004,7 +1005,7 @@ def generate_aggregation_query(self): raise NotImplementedError() def add_aggregation_data(self, payload): - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'interval_aggs' in payload_data: self.unwrap_interval_buckets(timestamp, None, payload_data['interval_aggs']['buckets']) elif 'bucket_aggs' in payload_data: @@ -1094,7 +1095,7 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou # add compound key to payload to allow alerts to trigger for every unique occurence compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join([unicode(value) for value in compound_value]) + match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) self.add_match(match_data) @@ -1140,7 +1141,7 @@ def add_aggregation_data(self, payload): We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from the aggregation's "value" key to determine spikes from aggregations """ - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'bucket_aggs' in payload_data: self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) else: diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6828b6829..cc5d52395 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -1,4 +1,4 @@ -$schema: http://json-schema.org/draft-04/schema# +$schema: http://json-schema.org/draft-07/schema# definitions: # Either a single string OR an array of strings @@ -11,6 +11,17 @@ definitions: type: [string, array] items: {type: [string, array]} + timedelta: &timedelta + type: object + additionalProperties: false + properties: + days: {type: number} + weeks: {type: number} + hours: {type: number} + minutes: {type: number} + seconds: {type: number} + milliseconds: {type: number} + timeFrame: &timeframe type: object additionalProperties: false @@ -203,6 +214,15 @@ properties: replace_dots_in_field_names: {type: boolean} scan_entire_timeframe: {type: boolean} + ### Kibana Discover App Link + generate_kibana_discover_url: {type: boolean} + kibana_discover_app_url: {type: string, format: uri} + kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_index_pattern_id: {type: string, minLength: 1} + kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} + kibana_discover_from_timedelta: *timedelta + kibana_discover_to_timedelta: *timedelta + # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} @@ -265,6 +285,10 @@ properties: slack_parse_override: {enum: [none, full]} slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} + slack_ca_certs: {type: string} + slack_attach_kibana_discover_url {type: boolean} + slack_kibana_discover_color {type: string} + slack_kibana_discover_title {type: string} ### Mattermost mattermost_webhook_url: *arrayOfString @@ -277,6 +301,20 @@ properties: mattermost_msg_pretext: {type: string} mattermost_msg_fields: *mattermostField + ## Opsgenie + opsgenie_details: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + ### PagerDuty pagerduty_service_key: {type: string} pagerduty_client_name: {type: string} diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index a9d889b28..ee467031b 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -1,27 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - +import argparse import copy import datetime import json import logging -import os import random import re import string import sys -import argparse import mock -import yaml -import elastalert.config -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rule_yaml +from elastalert.config import load_conf from elastalert.elastalert import ElastAlerter +from elastalert.util import EAException from elastalert.util import elasticsearch_client from elastalert.util import lookup_es_key from elastalert.util import ts_now @@ -30,6 +23,14 @@ logging.getLogger().setLevel(logging.INFO) logging.getLogger('elasticsearch').setLevel(logging.WARNING) +""" +Error Codes: + 1: Error connecting to ElasticSearch + 2: Error querying ElasticSearch + 3: Invalid Rule + 4: Missing/invalid timestamp +""" + def print_terms(terms, parent): """ Prints a list of flattened dictionary keys """ @@ -56,13 +57,17 @@ def test_file(self, conf, args): try: ElastAlerter.modify_rule_for_ES5(conf) + except EAException as ea: + print('Invalid filter provided:', str(ea), file=sys.stderr) + if args.stop_error: + exit(3) + return None except Exception as e: print("Error connecting to ElasticSearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: exit(1) return None - start_time = ts_now() - datetime.timedelta(days=args.days) end_time = ts_now() ts = conf.get('timestamp_field', '@timestamp') @@ -83,10 +88,11 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(3) return None num_hits = len(res['hits']['hits']) if not num_hits: + print("Didn't get any results.") return [] terms = res['hits']['hits'][0]['_source'] @@ -108,7 +114,7 @@ def test_file(self, conf, args): print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = res['count'] @@ -116,7 +122,7 @@ def test_file(self, conf, args): if args.formatted_output: self.formatted_output['hits'] = num_hits self.formatted_output['days'] = args.days - self.formatted_output['terms'] = terms.keys() + self.formatted_output['terms'] = list(terms.keys()) self.formatted_output['result'] = terms else: print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) @@ -152,7 +158,7 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = len(res['hits']['hits']) @@ -182,7 +188,7 @@ def mock_hits(self, rule, start, end, index, scroll=False): if field != '_id': if not any([re.match(incl.replace('*', '.*'), field) for incl in rule['include']]): fields_to_remove.append(field) - map(doc.pop, fields_to_remove) + list(map(doc.pop, fields_to_remove)) # Separate _source and _id, convert timestamps resp = [{'_source': doc, '_id': doc['_id']} for doc in docs] @@ -202,7 +208,7 @@ def mock_terms(self, rule, start, end, index, key, qk=None, size=None): if qk is None or doc[rule['query_key']] == qk: buckets.setdefault(doc[key], 0) buckets[doc[key]] += 1 - counts = buckets.items() + counts = list(buckets.items()) counts.sort(key=lambda x: x[1], reverse=True) if size: counts = counts[:size] @@ -224,8 +230,7 @@ def run_elastalert(self, rule, conf, args): # It is needed to prevent unnecessary initialization of unused alerters load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert - load_modules(rule, load_modules_args) - conf['rules'] = [rule] + conf['rules_loader'].load_modules(rule, load_modules_args) # If using mock data, make sure it's sorted and find appropriate time range timestamp_field = rule.get('timestamp_field', '@timestamp') @@ -240,7 +245,7 @@ def run_elastalert(self, rule, conf, args): except KeyError as e: print("All documents must have a timestamp and _id: %s" % (e), file=sys.stderr) if args.stop_error: - exit(1) + exit(4) return None # Create mock _id for documents if it's missing @@ -264,7 +269,7 @@ def get_id(): endtime = ts_to_dt(args.end) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.end)) - exit(1) + exit(4) else: endtime = ts_now() if args.start: @@ -272,7 +277,7 @@ def get_id(): starttime = ts_to_dt(args.start) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.start)) - exit(1) + exit(4) else: # if days given as command line argument if args.days > 0: @@ -291,13 +296,15 @@ def get_id(): conf['run_every'] = endtime - starttime # Instantiate ElastAlert to use mock config and special rule - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - if args.alert: - client = ElastAlerter(['--verbose']) - else: - client = ElastAlerter(['--debug']) + with mock.patch.object(conf['rules_loader'], 'get_hashes'): + with mock.patch.object(conf['rules_loader'], 'load') as load_rules: + load_rules.return_value = [rule] + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + if args.alert: + client = ElastAlerter(['--verbose']) + else: + client = ElastAlerter(['--debug']) # Replace get_hits_* functions to use mock data if args.json: @@ -327,56 +334,7 @@ def get_id(): if call[0][0] == 'elastalert_error': errors = True if errors and args.stop_error: - exit(1) - - def load_conf(self, rules, args): - """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), - for initializing rules. Also initializes rules. - - :return: the default rule configuration, a dictionary """ - if args.config is not None: - with open(args.config) as fh: - conf = yaml.load(fh, Loader=yaml.FullLoader) - else: - if os.path.isfile('config.yaml'): - with open('config.yaml') as fh: - conf = yaml.load(fh, Loader=yaml.FullLoader) - else: - conf = {} - - # Need to convert these parameters to datetime objects - for key in ['buffer_time', 'run_every', 'alert_time_limit', 'old_query_limit']: - if key in conf: - conf[key] = datetime.timedelta(**conf[key]) - - # Mock configuration. This specifies the base values for attributes, unless supplied otherwise. - conf_default = { - 'rules_folder': 'rules', - 'es_host': 'localhost', - 'es_port': 14900, - 'writeback_index': 'wb', - 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), - 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), - 'scroll_keepalive': '30s' - } - - for key in conf_default: - if key not in conf: - conf[key] = conf_default[key] - elastalert.config.base_config = copy.deepcopy(conf) - load_options(rules, conf, args.file) - - if args.formatted_output: - self.formatted_output['success'] = True - self.formatted_output['name'] = rules['name'] - else: - print("Successfully loaded %s\n" % (rules['name'])) - - return conf + exit(2) def run_rule_test(self): """ @@ -427,9 +385,33 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - rule_yaml = load_rule_yaml(args.file) + defaults = { + 'rules_folder': 'rules', + 'es_host': 'localhost', + 'es_port': 14900, + 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', + 'max_query_size': 10000, + 'alert_time_limit': {'hours': 24}, + 'old_query_limit': {'weeks': 1}, + 'run_every': {'minutes': 5}, + 'disable_rules_on_error': False, + 'buffer_time': {'minutes': 45}, + 'scroll_keepalive': '30s' + } + overwrites = { + 'rules_loader': 'file', + } + + # Set arguments that ElastAlerter needs + args.verbose = args.alert + args.debug = not args.alert + args.es_debug = False + args.es_debug_trace = False - conf = self.load_conf(rule_yaml, args) + conf = load_conf(args, defaults, overwrites) + rule_yaml = conf['rules_loader'].load_yaml(args.file) + conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: with open(args.json, 'r') as data_file: diff --git a/elastalert/util.py b/elastalert/util.py index 5f0828c9a..bbb0600ff 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -4,17 +4,33 @@ import logging import os import re +import sys import dateutil.parser -import dateutil.tz -from auth import Auth -from . import ElasticSearchClient +import pytz from six import string_types +from . import ElasticSearchClient +from .auth import Auth + logging.basicConfig() elastalert_logger = logging.getLogger('elastalert') +def get_module(module_name): + """ Loads a module and returns a specific object. + module_name should 'module.file.object'. + Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) + try: + module_path, module_class = module_name.rsplit('.', 1) + base_module = __import__(module_path, globals(), locals(), [module_class]) + module = getattr(base_module, module_class) + except (ImportError, AttributeError, ValueError) as e: + raise EAException("Could not import module %s: %s" % (module_name, e)).with_traceback(sys.exc_info()[2]) + return module + + def new_get_event_ts(ts_field): """ Constructs a lambda that may be called to extract the timestamp field from a given event. @@ -130,7 +146,7 @@ def ts_to_dt(timestamp): dt = dateutil.parser.parse(timestamp) # Implicitly convert local timestamps to UTC if dt.tzinfo is None: - dt = dt.replace(tzinfo=dateutil.tz.tzutc()) + dt = dt.replace(tzinfo=pytz.utc) return dt @@ -372,6 +388,15 @@ def build_es_conn_config(conf): return parsed_conf +def pytzfy(dt): + # apscheduler requires pytz timezone objects + # This function will replace a dateutil.tz one with a pytz one + if dt.tzinfo is not None: + new_tz = pytz.timezone(dt.tzinfo.tzname('Y is this even required??')) + return dt.replace(tzinfo=new_tz) + return dt + + def parse_duration(value): """Convert ``unit=num`` spec into a ``timedelta`` object.""" unit, num = value.split('=') @@ -386,7 +411,7 @@ def parse_deadline(value): def flatten_dict(dct, delim='.', prefix=''): ret = {} - for key, val in dct.items(): + for key, val in list(dct.items()): if type(val) == dict: ret.update(flatten_dict(val, prefix=prefix + key + delim)) else: @@ -417,9 +442,9 @@ def resolve_string(string, match, missing_text=''): string = string.format(**dd_match) break except KeyError as e: - if '{%s}' % e.message not in string: + if '{%s}' % str(e).strip("'") not in string: break - string = string.replace('{%s}' % e.message, '{_missing_value}') + string = string.replace('{%s}' % str(e).strip("'"), '{_missing_value}') return string diff --git a/example_rules/ssh-repeat-offender.yaml b/example_rules/ssh-repeat-offender.yaml new file mode 100644 index 000000000..27a439fcd --- /dev/null +++ b/example_rules/ssh-repeat-offender.yaml @@ -0,0 +1,61 @@ +# Rule name, must be unique +name: SSH abuse - reapeat offender + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 2 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + weeks: 1 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + - term: + rule_name: "SSH abuse" + +index: elastalert + +# When the attacker continues, send a new alert after x minutes +realert: + weeks: 4 + +query_key: + - match_body.source.ip + +include: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +alert_subject: "SSH abuse (repeat offender) on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - match_body.host.hostname + - kibana_link + +alert_text: |- + An reapeat offender has been active on {}. + + IP: {} + User: {} +alert_text_args: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +# The alert is use when a match is found +alert: + - slack + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml new file mode 100644 index 000000000..7af890784 --- /dev/null +++ b/example_rules/ssh.yaml @@ -0,0 +1,64 @@ +# Rule name, must be unique + name: SSH abuse (ElastAlert 3.0.1) - 2 + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 20 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + minutes: 60 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: +- query: + query_string: + query: "event.type:authentication_failure" + +index: auditbeat-* + +# When the attacker continues, send a new alert after x minutes +realert: + minutes: 1 + +query_key: + - source.ip + +include: + - host.hostname + - user.name + - source.ip + +include_match_in_root: true + +alert_subject: "SSH abuse on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - host.hostname + - kibana_link + +alert_text: |- + An attack on {} is detected. + The attacker looks like: + User: {} + IP: {} +alert_text_args: + - host.hostname + - user.name + - source.ip + +# The alert is use when a match is found +alert: + - debug + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/requirements-dev.txt b/requirements-dev.txt index 36daa0ebd..1cb67cb8e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +-r requirements.txt coverage flake8 pre-commit diff --git a/requirements.txt b/requirements.txt index 64b96986e..ce392cb18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,24 @@ +apscheduler>=3.3.0 aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 +cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 -jsonschema>=2.6.0 +jsonschema>=3.0.2 mock>=2.0.0 +prison>=0.1.2 +py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 -PyYAML>=3.12 +python-magic>=0.4.15 +PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 -twilio==6.0.0 thehive4py>=1.4.4 -python-magic>=0.4.15 -cffi>=1.11.5 -py-zabbix==1.1.3 \ No newline at end of file +twilio==6.0.0 diff --git a/setup.py b/setup.py index b14d345d9..ac6506ae2 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.39', + version='0.2.1', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', setup_requires='setuptools', license='Copyright 2014 Yelp', classifiers=[ - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], @@ -27,6 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']}, install_requires=[ + 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', @@ -35,9 +36,10 @@ 'elasticsearch>=7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', - 'jira>=1.0.10,<1.0.15', - 'jsonschema>=2.6.0,<3.0.0', + 'jira>=2.0.0', + 'jsonschema>=3.0.2', 'mock>=2.0.0', + 'prison>=0.1.2', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=3.12', diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 4564debee..5cd61ae75 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +import base64 import datetime import json import subprocess -from contextlib import nested import mock import pytest @@ -21,7 +21,7 @@ from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter from elastalert.alerts import StrideAlerter -from elastalert.config import load_modules +from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add from elastalert.util import ts_now @@ -35,7 +35,7 @@ def get_match_str(self, event): def test_basic_match_string(ea): ea.rules[0]['top_count_keys'] = ['username'] match = {'@timestamp': '1918-01-17', 'field': 'value', 'top_events_username': {'bob': 10, 'mallory': 5}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'anytest' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -44,32 +44,32 @@ def test_basic_match_string(ea): # Non serializable objects don't cause errors match['non-serializable'] = {open: 10} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) # unicode objects dont cause errors - match['snowman'] = u'☃' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + match['snowman'] = '☃' + alert_text = str(BasicMatchString(ea.rules[0], match)) # Pretty printed objects match.pop('non-serializable') match['object'] = {'this': {'that': [1, 2, "3"]}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) - assert '"this": {\n "that": [\n 1, \n 2, \n "3"\n ]\n }' in alert_text + alert_text = str(BasicMatchString(ea.rules[0], match)) + assert '"this": {\n "that": [\n 1,\n 2,\n "3"\n ]\n }' in alert_text ea.rules[0]['alert_text'] = 'custom text' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'anytest' not in alert_text ea.rules[0]['alert_text_type'] = 'alert_text_only' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' not in alert_text assert 'username' not in alert_text assert 'field: value' not in alert_text ea.rules[0]['alert_text_type'] = 'exclude_fields' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -83,8 +83,8 @@ def test_jira_formatted_match_string(ea): expected_alert_text_snippet = '{code}{\n' \ + tab + '"foo": {\n' \ + 2 * tab + '"bar": [\n' \ - + 3 * tab + '"one", \n' \ - + 3 * tab + '2, \n' \ + + 3 * tab + '"one",\n' \ + + 3 * tab + '2,\n' \ + 3 * tab + '"three"\n' \ + 2 * tab + ']\n' \ + tab + '}\n' \ @@ -95,7 +95,7 @@ def test_jira_formatted_match_string(ea): def test_email(): rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -158,9 +158,9 @@ def test_email_from_field(): def test_email_with_unicode_strings(): - rule = {'name': 'test alert', 'email': u'testing@test.test', 'from_addr': 'testfrom@test.test', + rule = {'name': 'test alert', 'email': 'testing@test.test', 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -170,7 +170,7 @@ def test_email_with_unicode_strings(): mock.call().ehlo(), mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), - mock.call().sendmail(mock.ANY, [u'testing@test.test'], mock.ANY), + mock.call().sendmail(mock.ANY, ['testing@test.test'], mock.ANY), mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -329,7 +329,7 @@ def test_email_with_args(): mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) - alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': u'☃'}}]) + alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': '☃'}}]) expected = [mock.call('localhost'), mock.call().ehlo(), mock.call().has_extn('STARTTLS'), @@ -340,7 +340,7 @@ def test_email_with_args(): body = mock_smtp.mock_calls[4][1][2] # Extract the MIME encoded message body - body_text = body.split('\n\n')[-1][:-1].decode('base64') + body_text = base64.b64decode(body.split('\n\n')[-1][:-1]).decode('utf-8') assert 'testing' in body_text assert '' in body_text @@ -380,9 +380,9 @@ def test_opsgenie_basic(): alert = OpsGenieAlerter(rule) alert.alert([{'@timestamp': '2014-10-31T00:00:00'}]) - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -406,9 +406,9 @@ def test_opsgenie_frequency(): assert alert.get_info()['recipients'] == rule['opsgenie_recipients'] - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -456,6 +456,261 @@ def test_opsgenie_default_alert_routing(): assert alert.get_info()['recipients'] == ['devops@test.com'] +def test_opsgenie_details_with_constant_value(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': 'Bar'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'message'}} + } + match = { + 'message': 'Bar', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_nested_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'nested.field'}} + } + match = { + 'nested': { + 'field': 'Bar' + }, + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_non_string_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Age': {'field': 'age'}, + 'Message': {'field': 'message'} + } + } + match = { + 'age': 10, + 'message': { + 'format': 'The cow goes %s!', + 'arg0': 'moo' + } + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': { + 'Age': '10', + 'Message': "{'format': 'The cow goes %s!', 'arg0': 'moo'}" + }, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_missing_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Message': {'field': 'message'}, + 'Missing': {'field': 'missing'} + } + } + match = { + 'message': 'Testing', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Message': 'Testing'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_environment_variable_replacement(environ): + environ.update({ + 'TEST_VAR': 'Bar' + }) + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': '$TEST_VAR'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = { @@ -478,10 +733,8 @@ def test_jira(): mock_priority = mock.Mock(id='5') - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = [] @@ -511,10 +764,8 @@ def test_jira(): # Search called if jira_bump_tickets rule['jira_bump_tickets'] = True - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -529,10 +780,8 @@ def test_jira(): # Remove a field if jira_ignore_in_title set rule['jira_ignore_in_title'] = 'test_term' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -545,10 +794,8 @@ def test_jira(): assert 'test_value' not in mock_jira.mock_calls[3][1][0] # Issue is still created if search_issues throws an exception - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.side_effect = JIRAError @@ -566,10 +813,8 @@ def test_jira(): # Check ticket is bumped if it is updated 4 days ago mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4)) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -584,10 +829,8 @@ def test_jira(): # Check ticket is bumped is not bumped if ticket is updated right now mock_issue.fields.updated = str(ts_now()) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -621,10 +864,8 @@ def test_jira(): mock_fields = [ {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}} ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -696,10 +937,8 @@ def test_jira_arbitrary_field_support(): }, ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -739,10 +978,8 @@ def test_jira_arbitrary_field_support(): # Reference an arbitrary string field that is not defined on the JIRA server rule['jira_nonexistent_field'] = 'nonexistent field value' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -757,10 +994,8 @@ def test_jira_arbitrary_field_support(): # Reference a watcher that does not exist rule['jira_watchers'] = 'invalid_watcher' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -902,7 +1137,8 @@ def test_ms_teams(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -938,7 +1174,8 @@ def test_ms_teams_uses_color_and_fixed_width_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -974,7 +1211,8 @@ def test_slack_uses_custom_title(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1004,7 +1242,7 @@ def test_slack_uses_custom_title(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True, + verify=False, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1019,7 +1257,8 @@ def test_slack_uses_custom_timeout(): 'alert': [], 'slack_timeout': 20 } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1049,7 +1288,7 @@ def test_slack_uses_custom_timeout(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True, + verify=False, timeout=20 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1062,7 +1301,8 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): 'slack_webhook_url': ['http://please.dontgohere.slack'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1092,7 +1332,7 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True, + verify=False, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1106,7 +1346,8 @@ def test_slack_uses_custom_slack_channel(): 'slack_channel_override': '#test-alert', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1136,7 +1377,7 @@ def test_slack_uses_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True, + verify=False, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1150,7 +1391,8 @@ def test_slack_uses_list_of_custom_slack_channel(): 'slack_channel_override': ['#test-alert', '#test-alert2'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1196,13 +1438,213 @@ def test_slack_uses_list_of_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True, + verify=False, timeout=10 ) assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) +def test_slack_attach_kibana_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_attach_kibana_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_title': 'Click to discover in Kibana', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_color': 'blue', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', @@ -1212,7 +1654,8 @@ def test_http_alerter_with_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1244,7 +1687,8 @@ def test_http_alerter_with_payload_all_values(): 'http_post_all_values': True, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1276,7 +1720,8 @@ def test_http_alerter_without_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1307,7 +1752,8 @@ def test_pagerduty_alerter(): 'pagerduty_client_name': 'ponies inc.', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1344,7 +1790,8 @@ def test_pagerduty_alerter_v2(): 'pagerduty_v2_payload_source': 'mysql.host.name', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1384,7 +1831,8 @@ def test_pagerduty_alerter_custom_incident_key(): 'pagerduty_incident_key': 'custom key', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1416,7 +1864,8 @@ def test_pagerduty_alerter_custom_incident_key_with_args(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1449,7 +1898,8 @@ def test_pagerduty_alerter_custom_alert_subject(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1483,7 +1933,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1519,7 +1970,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args_specifying_trigger(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1550,7 +2002,7 @@ def test_alert_text_kw(ea): 'field': 'field', } match = {'@timestamp': '1918-01-17', 'field': 'value'} - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) body = '{field} at {@timestamp}'.format(**match) assert body in alert_text @@ -1569,7 +2021,7 @@ def test_alert_text_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Priority: priority from rule' in alert_text assert 'Owner: the owner from rule' in alert_text @@ -1595,7 +2047,7 @@ def test_alert_text_kw_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Owner: the owner from rule' in alert_text assert 'Foo: foo from rule' in alert_text @@ -1644,7 +2096,8 @@ def test_stride_plain_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1689,7 +2142,8 @@ def test_stride_underline_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1734,7 +2188,8 @@ def test_stride_bold_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1779,7 +2234,8 @@ def test_stride_strong_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1824,7 +2280,8 @@ def test_stride_hyperlink(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1869,7 +2326,8 @@ def test_stride_html(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1921,7 +2379,8 @@ def test_hipchat_body_size_limit_text(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1948,7 +2407,8 @@ def test_hipchat_body_size_limit_html(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1965,7 +2425,7 @@ def test_alerta_no_auth(ea): 'name': 'Test Alerta rule!', 'alerta_api_url': 'http://elastalerthost:8080/api/alert', 'timeframe': datetime.timedelta(hours=1), - 'timestamp_field': u'@timestamp', + 'timestamp_field': '@timestamp', 'alerta_api_skip_ssl': True, 'alerta_attributes_keys': ["hostname", "TimestampEvent", "senderIP"], 'alerta_attributes_values': ["%(key)s", "%(logdate)s", "%(sender_ip)s"], @@ -1982,14 +2442,15 @@ def test_alerta_no_auth(ea): } match = { - u'@timestamp': '2014-10-10T00:00:00', + '@timestamp': '2014-10-10T00:00:00', # 'key': ---- missing field on purpose, to verify that simply the text is left empty # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty 'sender_ip': '1.1.1.1', 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -2043,7 +2504,8 @@ def test_alerta_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -2086,7 +2548,8 @@ def test_alerta_new_style(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) diff --git a/tests/base_test.py b/tests/base_test.py index 48f5d6dfc..0e57a2ff9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import contextlib import copy import datetime import json @@ -40,7 +39,7 @@ def generate_hits(timestamps, **kwargs): '_source': {'@timestamp': ts}, '_type': 'logs', '_index': 'idx'} - for key, item in kwargs.iteritems(): + for key, item in kwargs.items(): data['_source'][key] = item # emulate process_hits(), add metadata to _source for field in ['_id', '_type', '_index']: @@ -70,7 +69,7 @@ def test_init_rule(ea): # Simulate state of a rule just loaded from a file ea.rules[0]['minimum_starttime'] = datetime.datetime.now() new_rule = copy.copy(ea.rules[0]) - map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']) + list(map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime'])) # Properties are copied from ea.rules[0] ea.rules[0]['starttime'] = '2014-01-02T00:11:22' @@ -87,9 +86,9 @@ def test_init_rule(ea): def test_query(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], @@ -98,21 +97,21 @@ def test_query(ea): def test_query_sixsix(ea_sixsix): - ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea_sixsix.run_query(ea_sixsix.rules[0], START, END) - ea_sixsix.current_es.search.assert_called_with(body={ + ea_sixsix.thread_data.current_es.search.assert_called_with(body={ 'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) def test_query_with_fields(ea): ea.rules[0]['_source_enabled'] = False - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True, @@ -121,9 +120,9 @@ def test_query_with_fields(ea): def test_query_sixsix_with_fields(ea_sixsix): ea_sixsix.rules[0]['_source_enabled'] = False - ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea_sixsix.run_query(ea_sixsix.rules[0], START, END) - ea_sixsix.current_es.search.assert_called_with(body={ + ea_sixsix.thread_data.current_es.search.assert_called_with(body={ 'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'stored_fields': ['@timestamp']}, index='idx', @@ -134,11 +133,11 @@ def test_query_sixsix_with_fields(ea_sixsix): def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], @@ -149,14 +148,14 @@ def test_query_with_unix(ea): def test_query_sixsix_with_unix(ea_sixsix): ea_sixsix.rules[0]['timestamp_type'] = 'unix' ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unix - ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea_sixsix.run_query(ea_sixsix.rules[0], START, END) start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) - ea_sixsix.current_es.search.assert_called_with( + ea_sixsix.thread_data.current_es.search.assert_called_with( body={'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) @@ -164,11 +163,11 @@ def test_query_sixsix_with_unix(ea_sixsix): def test_query_with_unixms(ea): ea.rules[0]['timestamp_type'] = 'unixms' ea.rules[0]['dt_to_ts'] = dt_to_unixms - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], @@ -179,20 +178,20 @@ def test_query_with_unixms(ea): def test_query_sixsix_with_unixms(ea_sixsix): ea_sixsix.rules[0]['timestamp_type'] = 'unixms' ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unixms - ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea_sixsix.run_query(ea_sixsix.rules[0], START, END) start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) - ea_sixsix.current_es.search.assert_called_with( + ea_sixsix.thread_data.current_es.search.assert_called_with( body={'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) def test_no_hits(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 0 @@ -201,7 +200,7 @@ def test_no_terms_hits(ea): ea.rules[0]['use_terms_query'] = True ea.rules[0]['query_key'] = 'QWERTY' ea.rules[0]['doc_type'] = 'uiop' - ea.current_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_terms_data.call_count == 0 @@ -209,7 +208,7 @@ def test_no_terms_hits(ea): def test_some_hits(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -221,7 +220,7 @@ def test_some_hits_unix(ea): ea.rules[0]['ts_to_dt'] = unix_to_dt hits = generate_hits([dt_to_unix(START), dt_to_unix(END)]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = copy.deepcopy(hits) + ea.thread_data.current_es.search.return_value = copy.deepcopy(hits) ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -235,7 +234,7 @@ def _duplicate_hits_generator(timestamps, **kwargs): def test_duplicate_timestamps(ea): - ea.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') + ea.thread_data.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') ea.run_query(ea.rules[0], START, ts_to_dt('2014-01-01T00:00:00Z')) assert len(ea.rules[0]['type'].add_data.call_args_list[0][0][0]) == 3 @@ -248,7 +247,7 @@ def test_duplicate_timestamps(ea): def test_match(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -262,8 +261,8 @@ def test_run_rule_calls_garbage_collect(ea): end_time = '2014-09-26T12:00:00Z' ea.buffer_time = datetime.timedelta(hours=1) ea.run_every = datetime.timedelta(hours=1) - with contextlib.nested(mock.patch.object(ea.rules[0]['type'], 'garbage_collect'), - mock.patch.object(ea, 'run_query')) as (mock_gc, mock_get_hits): + with mock.patch.object(ea.rules[0]['type'], 'garbage_collect') as mock_gc, \ + mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], ts_to_dt(end_time), ts_to_dt(start_time)) # Running ElastAlert every hour for 12 hours, we should see self.garbage_collect called 12 times. @@ -318,7 +317,7 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 @@ -326,7 +325,7 @@ def test_match_with_module_from_pending(ea): # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() @@ -339,7 +338,7 @@ def test_match_with_module_with_agg(ea): ea.rules[0]['match_enhancements'] = [mod] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -353,7 +352,7 @@ def test_match_with_enhancements_first(ea): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) ea.rules[0]['run_enhancements_first'] = True hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch.object(ea, 'add_aggregated_alert') as add_alert: @@ -376,10 +375,10 @@ def test_agg_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10)) hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: # Aggregate first two, query over full range - mock_es.return_value = ea.current_es + mock_es.return_value = ea.thread_data.current_es ea.rules[0]['aggregate_by_match_time'] = True ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -405,9 +404,9 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -433,7 +432,7 @@ def test_agg_not_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] match_time = ts_to_dt('2014-09-26T12:55:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -462,7 +461,7 @@ def test_agg_cron(ea): ea.max_aggregation = 1337 hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00')) alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00')) @@ -500,7 +499,7 @@ def test_agg_no_writeback_connectivity(ea): run again, that they will be passed again to add_aggregated_alert """ hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45' hits = generate_hits([hit1, hit2, hit3]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': hit1}, {'@timestamp': hit2}, @@ -514,10 +513,10 @@ def test_agg_no_writeback_connectivity(ea): {'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3}, {'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}] - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], END, START) ea.add_aggregated_alert.assert_any_call({'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3}, ea.rules[0]) @@ -530,9 +529,9 @@ def test_agg_with_aggregation_key(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45'] match_time = ts_to_dt('2014-09-26T12:45:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: - mock_es.return_value = ea.current_es + mock_es.return_value = ea.thread_data.current_es with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -573,13 +572,13 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: - mock_es.return_value = ea.current_es + mock_es.return_value = ea.thread_data.current_es ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules assert mock_es.called_with(host='', port='') @@ -624,12 +623,12 @@ def test_silence(ea): def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] - hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'☃', those=4) - ea.current_es.search.return_value = hits + hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that='☃', those=4) + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] - assert call_args[0][0][0]['this,that,those'] == u'abc, ☃, 4' + assert call_args[0][0][0]['this,that,those'] == 'abc, ☃, 4' def test_silence_query_key(ea): @@ -667,7 +666,7 @@ def test_silence_query_key(ea): def test_realert(ea): hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)] matches = [{'@timestamp': x} for x in hits] - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.rules[0]['realert'] = datetime.timedelta(seconds=50) ea.rules[0]['type'].matches = matches @@ -754,7 +753,8 @@ def test_realert_with_nested_query_key(ea): def test_count(ea): ea.rules[0]['use_count_query'] = True ea.rules[0]['doc_type'] = 'doctype' - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count') as mock_hits: ea.run_rule(ea.rules[0], END, START) # Assert that es.count is run against every run_every timeframe between START and END @@ -766,8 +766,8 @@ def test_count(ea): end = start + ea.run_every query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) + mock_hits.assert_any_call(mock.ANY, start, end, mock.ANY) start = start + ea.run_every - ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -792,9 +792,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): def test_query_segmenting_reset_num_hits(ea): # Tests that num_hits gets reset every time run_query is run def assert_num_hits_reset(): - assert ea.num_hits == 0 - ea.num_hits += 10 - + assert ea.thread_data.num_hits == 0 + ea.thread_data.num_hits += 10 with mock.patch.object(ea, 'run_query') as mock_run_query: mock_run_query.side_effect = assert_num_hits_reset() ea.run_rule(ea.rules[0], END, START) @@ -911,7 +910,8 @@ def test_set_starttime(ea): assert ea.rules[0]['starttime'] == end - ea.run_every # Count query, with previous endtime - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count'): ea.run_rule(ea.rules[0], END, START) ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == END @@ -978,7 +978,7 @@ def test_kibana_dashboard(ea): url = ea.use_kibana_link(ea.rules[0], match) db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard']) found_filters = 0 - for filter_id, filter_dict in db['services']['filter']['list'].items(): + for filter_id, filter_dict in list(db['services']['filter']['list'].items()): if (filter_dict['field'] == 'foo' and filter_dict['query'] == '"cat"') or \ (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): found_filters += 1 @@ -996,8 +996,8 @@ def test_rule_changes(ea): 'rules/rule3.yaml': 'XXX', 'rules/rule2.yaml': '!@#$'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] mock_hashes.return_value = new_hashes @@ -1017,8 +1017,8 @@ def test_rule_changes(ea): # A new rule with a conflicting name wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} @@ -1031,10 +1031,9 @@ def test_rule_changes(ea): # A new rule with is_enabled=False wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, - 'rule_file': 'rules/rule4.yaml'} + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 3 @@ -1043,13 +1042,22 @@ def test_rule_changes(ea): # An old rule which didn't load gets reloaded new_hashes = copy.copy(new_hashes) new_hashes['rules/rule4.yaml'] = 'qwerty' - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 4 + # Disable a rule by removing the file + new_hashes.pop('rules/rule4.yaml') + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_hashes.return_value = new_hashes + ea.load_rule_changes() + ea.scheduler.remove_job.assert_called_with(job_id='rule4') + def test_strf_index(ea): """ Test that the get_index function properly generates indexes spanning days """ @@ -1080,9 +1088,9 @@ def test_count_keys(ea): 'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, {'aggregations': {'filtered': { 'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] - ea.current_es.deprecated_search.side_effect = buckets + ea.thread_data.current_es.deprecated_search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) - calls = ea.current_es.deprecated_search.call_args_list + calls = ea.thread_data.current_es.deprecated_search.call_args_list assert calls[0][1]['search_type'] == 'count' assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, 'min_doc_count': 1} @@ -1115,7 +1123,7 @@ def test_exponential_realert(ea): for args in test_values: ea.silence_cache[ea.rules[0]['name']] = (args[1], args[2]) next_alert, exponent = ea.next_alert_time(ea.rules[0], ea.rules[0]['name'], args[0]) - assert exponent == next_res.next() + assert exponent == next(next_res) def test_wait_until_responsive(ea): @@ -1184,7 +1192,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ @@ -1208,7 +1216,7 @@ def mock_loop(): ea.stop() with mock.patch.object(ea, 'sleep_for', return_value=None): - with mock.patch.object(ea, 'run_all_rules') as mock_run: + with mock.patch.object(ea, 'sleep_for') as mock_run: mock_run.side_effect = mock_loop() start_thread = threading.Thread(target=ea.start) # Set as daemon to prevent a failed test from blocking exit @@ -1267,8 +1275,8 @@ def test_uncaught_exceptions(ea): # Changing the file should re-enable it ea.rule_hashes = {'blah.yaml': 'abc'} new_hashes = {'blah.yaml': 'def'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [ea.disabled_rules[0]] mock_hashes.return_value = new_hashes ea.load_rule_changes() diff --git a/tests/conftest.py b/tests/conftest.py index 6b9db2894..2b547ba41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import datetime - import logging -import mock import os + +import mock import pytest import elastalert.elastalert @@ -113,6 +113,14 @@ def writeback_index_side_effect(index, doc_type): self.resolve_writeback_index = mock.Mock(side_effect=writeback_index_side_effect) +class mock_rule_loader(object): + def __init__(self, conf): + self.base_config = conf + self.load = mock.Mock() + self.get_hashes = mock.Mock() + self.load_configuration = mock.Mock() + + class mock_ruletype(object): def __init__(self): self.add_data = mock.Mock() @@ -149,32 +157,40 @@ def ea(): 'max_query_size': 10000, 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts, - '_source_enabled': True}] + '_source_enabled': True, + 'run_every': datetime.timedelta(seconds=15)}] conf = {'rules_folder': 'rules', 'run_every': datetime.timedelta(minutes=10), 'buffer_time': datetime.timedelta(minutes=5), 'alert_time_limit': datetime.timedelta(hours=24), 'es_host': 'es', 'es_port': 14900, - 'writeback_index': writeback_index, + 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} elastalert.util.elasticsearch_client = mock_es_client + conf['rules_loader'] = mock_rule_loader(conf) elastalert.elastalert.elasticsearch_client = mock_es_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + with mock.patch('elastalert.elastalert.BackgroundScheduler'): load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() - ea.writeback_es.search.return_value = {'hits': {'hits': []}} + ea.writeback_es.search.return_value = {'hits': {'hits': []}, 'total': 0} ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} - ea.writeback_es.index.return_value = {'_id': 'ABCD'} + ea.writeback_es.index.return_value = {'_id': 'ABCD', 'created': True} ea.current_es = mock_es_client('', '') + ea.thread_data.current_es = ea.current_es + ea.thread_data.num_hits = 0 + ea.thread_data.num_dupes = 0 return ea @@ -203,16 +219,20 @@ def ea_sixsix(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': writeback_index, + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} + conf['rules_loader'] = mock_rule_loader(conf) elastalert.elastalert.elasticsearch_client = mock_es_sixsix_client elastalert.util.elasticsearch_client = mock_es_sixsix_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + with mock.patch('elastalert.elastalert.BackgroundScheduler'): load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} ea_sixsix = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea_sixsix.rules[0]['type'] = mock_ruletype() ea_sixsix.rules[0]['alert'] = [mock_alert()] @@ -228,7 +248,7 @@ def ea_sixsix(): def environ(): """py.test fixture to get a fresh mutable environment.""" old_env = os.environ - new_env = dict(old_env.items()) + new_env = dict(list(old_env.items())) os.environ = new_env yield os.environ os.environ = old_env diff --git a/tests/create_index_test.py b/tests/create_index_test.py index ba306aee5..47a6247dc 100644 --- a/tests/create_index_test.py +++ b/tests/create_index_test.py @@ -18,36 +18,36 @@ def test_read_default_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_5_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5) assert es_mapping in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_6_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) def test_read_default_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings() assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_5_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(5) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_6_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(6) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index b867d3543..308356c25 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -4,14 +4,13 @@ import time import dateutil -import mock import pytest import elastalert.create_index import elastalert.elastalert from elastalert import ElasticSearchClient -from elastalert.util import ts_to_dt, dt_to_ts, build_es_conn_config -from tests.conftest import mock_ruletype, mock_alert, mock_es_client +from elastalert.util import build_es_conn_config +from tests.conftest import ea # noqa: F401 test_index = 'test_index' @@ -26,49 +25,6 @@ def es_client(): return ElasticSearchClient(es_conn_config) -@pytest.fixture -def ea(): - rules = [{'es_host': '', - 'es_port': 14900, - 'name': 'anytest', - 'index': 'idx', - 'filter': [], - 'include': ['@timestamp'], - 'aggregation': datetime.timedelta(0), - 'realert': datetime.timedelta(0), - 'processed_hits': {}, - 'timestamp_field': '@timestamp', - 'match_enhancements': [], - 'rule_file': 'blah.yaml', - 'max_query_size': 10000, - 'ts_to_dt': ts_to_dt, - 'dt_to_ts': dt_to_ts, - '_source_enabled': True}] - conf = {'rules_folder': 'rules', - 'run_every': datetime.timedelta(minutes=10), - 'buffer_time': datetime.timedelta(minutes=5), - 'alert_time_limit': datetime.timedelta(hours=24), - 'es_host': es_host, - 'es_port': es_port, - 'es_conn_timeout': es_timeout, - 'writeback_index': test_index, - 'rules': rules, - 'max_query_size': 10000, - 'old_query_limit': datetime.timedelta(weeks=1), - 'disable_rules_on_error': False, - 'scroll_keepalive': '30s'} - elastalert.elastalert.elasticsearch_client = mock_es_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) - ea.rules[0]['type'] = mock_ruletype() - ea.rules[0]['alert'] = [mock_alert()] - ea.writeback_es = es_client() - ea.current_es = mock_es_client('', '') - return ea - - @pytest.mark.elasticsearch class TestElasticsearch(object): # TODO perform teardown removing data inserted into Elasticsearch @@ -78,9 +34,9 @@ class TestElasticsearch(object): def test_create_indices(self, es_client): elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index) indices_mappings = es_client.indices.get_mapping(test_index + '*') - print('-' * 50) - print(json.dumps(indices_mappings, indent=2)) - print('-' * 50) + print(('-' * 50)) + print((json.dumps(indices_mappings, indent=2))) + print(('-' * 50)) if es_client.is_atleastsix(): assert test_index in indices_mappings assert test_index + '_error' in indices_mappings @@ -94,7 +50,8 @@ def test_create_indices(self, es_client): assert 'silence' in indices_mappings[test_index]['mappings'] assert 'past_elastalert' in indices_mappings[test_index]['mappings'] - def test_aggregated_alert(self, ea): + @pytest.mark.usefixtures("ea") + def test_aggregated_alert(self, ea, es_client): # noqa: F811 match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) ea.rules[0]['aggregate_by_match_time'] = True @@ -102,6 +59,7 @@ def test_aggregated_alert(self, ea): 'num_hits': 0, 'num_matches': 3 } + ea.writeback_es = es_client res = ea.add_aggregated_alert(match, ea.rules[0]) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -112,9 +70,11 @@ def test_aggregated_alert(self, ea): # Now lets find the pending aggregated alert assert ea.find_pending_aggregate_alert(ea.rules[0]) - def test_silenced(self, ea): + @pytest.mark.usefixtures("ea") + def test_silenced(self, ea, es_client): # noqa: F811 until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) + ea.writeback_es = es_client res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -127,7 +87,8 @@ def test_silenced(self, ea): # Now lets check if our rule is reported as silenced assert ea.is_silenced(ea.rules[0]['name']) - def test_get_hits(self, ea, es_client): + @pytest.mark.usefixtures("ea") + def test_get_hits(self, ea, es_client): # noqa: F811 start = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) end = start + datetime.timedelta(days=1) ea.current_es = es_client @@ -135,5 +96,7 @@ def test_get_hits(self, ea, es_client): ea.rules[0]['five'] = True else: ea.rules[0]['five'] = False + ea.thread_data.current_es = ea.current_es hits = ea.get_hits(ea.rules[0], start, end, test_index) + assert isinstance(hits, list) diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py new file mode 100644 index 000000000..f06fe4e0c --- /dev/null +++ b/tests/kibana_discover_test.py @@ -0,0 +1,858 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta +import pytest + +from elastalert.kibana_discover import generate_kibana_discover_url + + +@pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_version(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_app_url(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_invalid_kibana_version(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '4.5', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitution(environ): + environ.update({ + 'KIBANA_HOST': 'kibana', + 'KIBANA_PORT': '5601', + }) + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_from_timedelta(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A10%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_to_timedelta(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A50%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T04%3A10%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A50%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_custom_columns(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'kibana_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28level%2Cmessage%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_single_filter(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_multiple_filters(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B' # value start + + '%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D' + + '%5D%7D%27' # value end + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_int_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3A200%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3A%27200%27' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # reponse start + + 'query%3A200%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_str_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_null_query_key_value(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': None + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_missing_query_key_value(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_compound_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # geo.src filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3ACA%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3ACA' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.src%3A%28' # reponse start + + 'query%3ACA%2C' + + 'type%3Aphrase' + + '%29' # geo.src end + + '%29' # match end + + '%29' # query end + + '%29%2C' # geo.src filter end + + + '%28' # geo.dest filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3AUS%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3AUS' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3AUS%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # geo.dest filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_filter_and_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29%2C' # filter end + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'status%3A%28' # status start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # status end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl diff --git a/tests/config_test.py b/tests/loaders_test.py similarity index 50% rename from tests/config_test.py rename to tests/loaders_test.py index 25e1062b2..bb8d3d873 100644 --- a/tests/config_test.py +++ b/tests/loaders_test.py @@ -1,27 +1,24 @@ # -*- coding: utf-8 -*- import copy import datetime +import os import mock -import os import pytest import elastalert.alerts import elastalert.ruletypes -from elastalert.config import get_file_paths -from elastalert.config import load_configuration -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rules +from elastalert.config import load_conf +from elastalert.loaders import FileRulesLoader from elastalert.util import EAException -from elastalert.config import import_rules test_config = {'rules_folder': 'test_folder', 'run_every': {'minutes': 10}, 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, @@ -49,15 +46,16 @@ def test_import_rules(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['type'] = 'testing.test.RuleType' - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'load_yaml') as mock_open: mock_open.return_value = test_rule_copy # Test that type is imported - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' assert mock_import.call_args_list[0][0][3] == ['RuleType'] @@ -65,14 +63,15 @@ def test_import_rules(): test_rule_copy = copy.deepcopy(test_rule) mock_open.return_value = test_rule_copy test_rule_copy['alert'] = 'testing2.test2.Alerter' - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.alerts - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' assert mock_import.call_args_list[0][0][3] == ['Alerter'] def test_import_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -83,9 +82,9 @@ def test_import_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -95,10 +94,11 @@ def test_import_import(): assert rules['filter'] == import_rule['filter'] # check global import_rule dependency - assert import_rules == {'blah.yaml': ['importme.ymlt']} + assert rules_loader.import_rules == {'blah.yaml': ['importme.ymlt']} def test_import_absolute_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -109,9 +109,9 @@ def test_import_absolute_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('/importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -124,6 +124,7 @@ def test_import_absolute_import(): def test_import_filter(): # Check that if a filter is specified the rules are merged: + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -134,13 +135,14 @@ def test_import_filter(): 'filter': [{'term': {'ratchet': 'clank'}}], } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert rules['filter'] == [{'term': {'ratchet': 'clank'}}, {'term': {'key': 'value'}}] def test_load_inline_alert_rule(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['alert'] = [ { @@ -155,34 +157,75 @@ def test_load_inline_alert_rule(): } ] test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [test_config_copy, test_rule_copy] - load_modules(test_rule_copy) + rules_loader.load_modules(test_rule_copy) assert isinstance(test_rule_copy['alert'][0], elastalert.alerts.EmailAlerter) assert isinstance(test_rule_copy['alert'][1], elastalert.alerts.EmailAlerter) assert 'foo@bar.baz' in test_rule_copy['alert'][0].rule['email'] assert 'baz@foo.bar' in test_rule_copy['alert'][1].rule['email'] +def test_file_rules_loader_get_names_recursive(): + conf = {'scan_subdirectories': True, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), + ('root/folder_a', (), ('a.yaml', 'ab.yaml')), + ('root/folder_b', (), ('b.yaml',))) + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = walk_paths + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/rule.yaml' in paths + assert 'root/folder_a/a.yaml' in paths + assert 'root/folder_a/ab.yaml' in paths + assert 'root/folder_b/b.yaml' in paths + assert len(paths) == 4 + + +def test_file_rules_loader_get_names(): + # Check for no subdirectory + conf = {'scan_subdirectories': False, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + files = ['badfile', 'a.yaml', 'b.yaml'] + + with mock.patch('os.listdir') as mock_list: + with mock.patch('os.path.isfile') as mock_path: + mock_path.return_value = True + mock_list.return_value = files + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/a.yaml' in paths + assert 'root/b.yaml' in paths + assert len(paths) == 2 + + def test_load_rules(): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) - assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) - assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) - assert isinstance(rules['run_every'], datetime.timedelta) - for included_key in ['comparekey', 'testkey', '@timestamp']: - assert included_key in rules['rules'][0]['include'] - - # Assert include doesn't contain duplicates - assert rules['rules'][0]['include'].count('@timestamp') == 1 - assert rules['rules'][0]['include'].count('comparekey') == 1 + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) + assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) + assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) + assert isinstance(rules['run_every'], datetime.timedelta) + for included_key in ['comparekey', 'testkey', '@timestamp']: + assert included_key in rules['rules'][0]['include'] + + # Assert include doesn't contain duplicates + assert rules['rules'][0]['include'].count('@timestamp') == 1 + assert rules['rules'][0]['include'].count('comparekey') == 1 def test_load_default_host_port(): @@ -190,16 +233,19 @@ def test_load_default_host_port(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - # Assert include doesn't contain duplicates - assert rules['es_port'] == 12345 - assert rules['es_host'] == 'elasticsearch.test' + # Assert include doesn't contain duplicates + assert rules['es_port'] == 12345 + assert rules['es_host'] == 'elasticsearch.test' def test_load_ssl_env_false(): @@ -207,15 +253,18 @@ def test_load_ssl_env_false(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is False + assert rules['use_ssl'] is False def test_load_ssl_env_true(): @@ -223,15 +272,18 @@ def test_load_ssl_env_true(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is True + assert rules['use_ssl'] is True def test_load_url_prefix_env(): @@ -239,68 +291,103 @@ def test_load_url_prefix_env(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['es_url_prefix'] == 'es/' + assert rules['es_url_prefix'] == 'es/' def test_load_disabled_rules(): test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['is_enabled'] = False test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + # The rule is not loaded for it has "is_enabled=False" + assert len(rules['rules']) == 0 - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - # The rule is not loaded for it has "is_enabled=False" - assert len(rules['rules']) == 0 + +def test_raises_on_missing_config(): + optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') + test_rule_copy = copy.deepcopy(test_rule) + for key in list(test_rule_copy.keys()): + test_rule_copy = copy.deepcopy(test_rule) + test_config_copy = copy.deepcopy(test_config) + test_rule_copy.pop(key) + + # Non required keys + if key in optional_keys: + continue + + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [('', [], ['testrule.yaml'])] + with pytest.raises(EAException, message='key %s should be required' % key): + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) def test_compound_query_key(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('use_count_query') test_rule_copy['query_key'] = ['field1', 'field2'] - load_options(test_rule_copy, test_config, 'filename.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') assert 'field1' in test_rule_copy['include'] assert 'field2' in test_rule_copy['include'] assert test_rule_copy['query_key'] == 'field1,field2' assert test_rule_copy['compound_query_key'] == ['field1', 'field2'] -def test_name_inference(): +def test_query_key_with_single_value(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) - test_rule_copy.pop('name') - load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') - assert test_rule_copy['name'] == 'msmerc woz ere' + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = ['field1'] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'field1' in test_rule_copy['include'] + assert test_rule_copy['query_key'] == 'field1' + assert 'compound_query_key' not in test_rule_copy -def test_raises_on_missing_config(): - optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') +def test_query_key_with_no_values(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): - test_rule_copy = copy.deepcopy(test_rule) - test_config_copy = copy.deepcopy(test_config) - test_rule_copy.pop(key) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = [] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'query_key' not in test_rule_copy + assert 'compound_query_key' not in test_rule_copy - # Non required keys - if key in optional_keys: - continue - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - with pytest.raises(EAException, message='key %s should be required' % key): - rule = load_rules(test_args) - print(rule) +def test_name_inference(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('name') + rules_loader.load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') + assert test_rule_copy['name'] == 'msmerc woz ere' def test_raises_on_bad_generate_kibana_filters(): @@ -319,48 +406,35 @@ def test_raises_on_bad_generate_kibana_filters(): # Test that all the good filters work, but fail with a bad filter added for good in good_filters: + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['filter'] = good - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.return_value = test_rule_copy - load_configuration('blah', test_config) + rules_loader.load_configuration('blah', test_config) for bad in bad_filters: test_rule_copy['filter'] = good + bad with pytest.raises(EAException): - load_configuration('blah', test_config) - - -def test_get_file_paths_recursive(): - conf = {'scan_subdirectories': True, 'rules_folder': 'root'} - walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), - ('root/folder_a', (), ('a.yaml', 'ab.yaml')), - ('root/folder_b', (), ('b.yaml',))) - with mock.patch('os.walk') as mock_walk: - mock_walk.return_value = walk_paths - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/rule.yaml' in paths - assert 'root/folder_a/a.yaml' in paths - assert 'root/folder_a/ab.yaml' in paths - assert 'root/folder_b/b.yaml' in paths - assert len(paths) == 4 - + rules_loader.load_configuration('blah', test_config) -def test_get_file_paths(): - # Check for no subdirectory - conf = {'scan_subdirectories': False, 'rules_folder': 'root'} - files = ['badfile', 'a.yaml', 'b.yaml'] - with mock.patch('os.listdir') as mock_list: - with mock.patch('os.path.isfile') as mock_path: - mock_path.return_value = True - mock_list.return_value = files - paths = get_file_paths(conf) +def test_kibana_discover_from_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_from_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_from_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_from_timedelta'] == datetime.timedelta(minutes=2) - paths = [p.replace(os.path.sep, '/') for p in paths] - assert 'root/a.yaml' in paths - assert 'root/b.yaml' in paths - assert len(paths) == 2 +def test_kibana_discover_to_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_to_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) diff --git a/tox.ini b/tox.ini index b8c80496a..71099e17c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py27,docs +envlist = py36,docs [testenv] deps = -rrequirements-dev.txt @@ -27,4 +27,4 @@ norecursedirs = .* virtualenv_run docs build venv env deps = {[testenv]deps} sphinx==1.6.6 changedir = docs -commands = sphinx-build -b html -d build/doctrees source build/html +commands = sphinx-build -b html -d build/doctrees -W source build/html