Merge branch 'master' of https://github.com/Yelp/elastalert into add-…

…support-for-alertmanager

.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+
+[*.py]
+indent_style = space
+indent_size = 4
+
+[Makefile]
+indent_style = tab
+
+[{*.json,*.yml,*.yaml}]
+indent_style = space
+indent_size = 2

.travis.yml

@@ -1,12 +1,35 @@
 language: python
 python:
-- '2.7'
+- '3.6'
 env:
 - TOXENV=docs
-- TOXENV=py27
+- TOXENV=py36
 install:
 - pip install tox
-script: make test
+- >
+  if [[ -n "${ES_VERSION}" ]] ; then
+    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
+    mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1
+    ./elasticsearch-${ES_VERSION}/bin/elasticsearch &
+  fi
+script:
+- >
+  if [[ -n "${ES_VERSION}" ]] ; then
+    wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200
+    make test-elasticsearch
+  else
+    make test
+  fi
+jobs:
+  include:
+    - stage: 'Elasticsearch test'
+      env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64
+    - env: TOXENV=py36 ES_VERSION=6.6.2
+    - env: TOXENV=py36 ES_VERSION=6.3.2
+    - env: TOXENV=py36 ES_VERSION=6.2.4
+    - env: TOXENV=py36 ES_VERSION=6.0.1
+    - env: TOXENV=py36 ES_VERSION=5.6.16
+
 deploy:
   provider: pypi
   user: yelplabs

Makefile

@@ -16,6 +16,9 @@ install-hooks:
 test:
 	tox
 
+test-elasticsearch:
+	tox -- --runelasticsearch
+
 test-docker:
 	docker-compose --project-name elastalert build tox
 	docker-compose --project-name elastalert run tox

README.md

@@ -1,5 +1,5 @@
-[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert)
-[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert)
+Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported.
+
 [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert)
 [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
@@ -42,27 +42,21 @@ Currently, we have built-in support for the following alert types:
 - Email
 - JIRA
 - OpsGenie
-- Stomp
 - Commands
-- AWS SNS
 - HipChat
-- Stride
 - MS Teams
 - Slack
-- Mattermost
+- Telegram
+- GoogleChat
+- AWS SNS
+- VictorOps
 - PagerDuty
 - PagerTree
 - Exotel
 - Twilio
-- VictorOps
-- Telegram
-- GoogleChat
 - Gitter
-- ServiceNow
 - Line Notify
-- Alerta
-- HTTP POST
-- theHive
+- Zabbix
 - Alertmanager
 
 Additional rule types and alerts can be easily imported or written.

changelog.md

@@ -1,5 +1,22 @@
 # Change Log
 
+# v0.2.1
+
+### Fixed
+- Fixed an AttributeError introduced in 0.2.0
+
+# v0.2.0
+
+- Switched to Python 3
+
+### Added
+- Add rule loader class for customized rule loading
+- Added thread based rules and limit_execution
+- Run_every can now be customized per rule
+
+### Fixed
+- Various small fixes
+
 # v0.1.39
 
 ### Added

config.yaml.example

@@ -57,6 +57,7 @@ es_port: 9200
 # This can be a unmapped index, but it is recommended that you run
 # elastalert-create-index to set a mapping
 writeback_index: elastalert_status
+writeback_alias: elastalert_alerts
 
 # If an alert fails for some reason, ElastAlert will retry
 # sending the alert until this time period has elapsed

docs/source/elastalert.rst

@@ -134,9 +134,12 @@ The environment variable ``ES_USE_SSL`` will override this field.
 
 ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``.
 
+``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes.
+Defaults to ``FileRulesLoader`` if not set.
+
 ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all
 files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload
-or remove rules based on their respective config files.
+or remove rules based on their respective config files. (only required when using ``FileRulesLoader``).
 
 ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true``
 
@@ -149,7 +152,11 @@ configuration.
 
 ``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. The
 default is 10,000, and if you expect to get near this number, consider using ``use_count_query`` for the rule. If this
-limit is reached, ElastAlert will `scroll <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html>`_ through pages the size of ``max_query_size`` until processing all results.
+limit is reached, ElastAlert will `scroll <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html>`_
+using the size of ``max_query_size`` through the set amount of pages, when ``max_scrolling_count`` is set or until processing all results.
+
+``max_scrolling_count``: The maximum amount of pages to scroll through. The default is ``0``, which means the scrolling has no limit.
+For example if this value is set to ``5`` and the ``max_query_size`` is set to ``10000`` then ``50000`` documents will be downloaded at most.
 
 ``scroll_keepalive``: The maximum time (formatted in `Time Units <https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#time-units>`_) the scrolling context should be kept alive. Avoid using high values as it abuses resources in Elasticsearch, but be mindful to allow sufficient time to finish processing all the results.
 
@@ -164,6 +171,8 @@ from that time, unless it is older than ``old_query_limit``, in which case it wi
 will upload a traceback message to ``elastalert_metadata`` and if ``notify_email`` is set, send an email notification. The
 rule will no longer be run until either ElastAlert restarts or the rule file has been modified. This defaults to True.
 
+``show_disabled_rules``: If true, ElastAlert show the disable rules' list when finishes the execution. This defaults to True.
+
 ``notify_email``: An email address, or list of email addresses, to which notification emails will be sent. Currently,
 only an uncaught exception will send a notification email. The from address, SMTP host, and reply-to header can be set
 using ``from_addr``, ``smtp_host``, and ``email_reply_to`` options, respectively. By default, no emails will be sent.

docs/source/index.rst

@@ -19,6 +19,7 @@ Contents:
    recipes/adding_alerts
    recipes/writing_filters
    recipes/adding_enhancements
+   recipes/adding_loaders
    recipes/signing_requests
 
 Indices and Tables

docs/source/recipes/adding_loaders.rst

@@ -0,0 +1,85 @@
+.. _loaders:
+
+Rules Loaders
+========================
+
+RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to
+gather rules for a particular source. Your RulesLoader needs to implement three member functions, and
+will look something like this:
+
+.. code-block:: python
+
+    class AwesomeNewRulesLoader(RulesLoader):
+        def get_names(self, conf, use_rule=None):
+            ...
+        def get_hashes(self, conf, use_rule=None):
+            ...
+        def get_yaml(self, rule):
+            ...
+
+You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a
+python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``.
+
+Example
+-------
+
+As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a
+modules folder for the loader in the ElastAlert directory.
+
+.. code-block:: console
+
+    $ mkdir elastalert_modules
+    $ cd elastalert_modules
+    $ touch __init__.py
+
+Now, in a file named ``mongo_loader.py``, add
+
+.. code-block:: python
+
+    from pymongo import MongoClient
+    from elastalert.loaders import RulesLoader
+    import yaml
+
+    class MongoRulesLoader(RulesLoader):
+        def __init__(self, conf):
+            super(MongoRulesLoader, self).__init__(conf)
+            self.client = MongoClient(conf['mongo_url'])
+            self.db = self.client[conf['mongo_db']]
+            self.cache = {}
+
+        def get_names(self, conf, use_rule=None):
+            if use_rule:
+                return [use_rule]
+
+            rules = []
+            self.cache = {}
+            for rule in self.db.rules.find():
+                self.cache[rule['name']] = yaml.load(rule['yaml'])
+                rules.append(rule['name'])
+
+            return rules
+
+        def get_hashes(self, conf, use_rule=None):
+            if use_rule:
+                return [use_rule]
+
+            hashes = {}
+            self.cache = {}
+            for rule in self.db.rules.find():
+                self.cache[rule['name']] = rule['yaml']
+                hashes[rule['name']] = rule['hash']
+
+            return hashes
+
+        def get_yaml(self, rule):
+            if rule in self.cache:
+                return self.cache[rule]
+
+            self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml'])
+            return self.cache[rule]
+
+Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the
+default FileRulesLoader, so in your ``elastalert.conf`` file::
+
+    rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader"
+

docs/source/recipes/writing_filters.rst

@@ -56,22 +56,23 @@ Note that a term query may not behave as expected if a field is analyzed. By def
 a field that appears to have the value "foo bar", unless it is not analyzed. Conversely, a term query for "foo" will match analyzed strings "foo bar" and "foo baz". For full text
 matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html
 
-terms
+`terms <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html>`_
 *****
 
+
+
 Terms allows for easy combination of multiple term filters::
 
     filter:
     - terms:
-        field: ["value1", "value2"]
+        field: ["value1", "value2"] # value1 OR value2
 
-Using the minimum_should_match option, you can define a set of term filters of which a certain number must match::
+You can also match on multiple fields::
 
     - terms:
         fieldX: ["value1", "value2"]
         fieldY: ["something", "something_else"]
         fieldZ: ["foo", "bar", "baz"]
-        minimum_should_match: 2
 
 wildcard
 ********

docs/source/ruletypes.rst

@@ -1869,7 +1869,7 @@ The alerter requires the following options:
 
 Optional:
 
-``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents thoughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert.
+``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents throughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert.
 
 ``victorops_entity_display_name``: Human-readable name of alerting entity to summarize incidents without affecting the life-cycle workflow.
 
@@ -2037,6 +2037,8 @@ Optional:
 
 ``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here.
 
+``http_post_headers``: Key:value pairs of headers to be sent as part of the request.
+
 ``http_post_proxy``: URL of proxy, if required.
 
 ``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False.
@@ -2051,6 +2053,8 @@ Example usage::
       ip: clientip
     http_post_static_payload:
       apikey: abc123
+    http_post_headers:
+      authorization: Basic 123dr3234
 
 
 Alerter
@@ -2128,7 +2132,7 @@ See https://prometheus.io/docs/alerting/clients/ for more details about the Aler
 
 Required:
 
-``alertmanager_host``: The host pointing to the Alertmanager.
+``alertmanager_hosts``: The list of hosts pointing to the Alertmanager.
 
 Optional:
 
@@ -2152,7 +2156,8 @@ Example usage::
 
     alert:
     - alertmanager:
-        alertmanager_host: http://localhost:9093
+        alertmanager_hosts:
+        - http://localhost:9093
         alertmanager_alertname: Title
         alertmanager_annotations:
             severity: error
@@ -2162,3 +2167,16 @@ Example usage::
             namespace: kubernetes.namespace_name
             app: kubernetes.labels.k8s-app
             pod_name: kubernetes.pod_name
+
+
+Zabbix
+~~~~~~~~~~~
+
+Zabbix will send notification to a Zabbix server. The item in the host specified receive a 1 value for each hit. For example, if the elastic query produce 3 hits in the last execution of elastalert, three '1' (integer) values will be send from elastalert to Zabbix Server. If the query have 0 hits, any value will be sent.
+
+Required:
+
+``zbx_sender_host``: The address where zabbix server is running.
+``zbx_sender_port``: The port where zabbix server is listenning.
+``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert.
+``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert.

docs/source/running_elastalert.rst

@@ -8,7 +8,7 @@ Requirements
 
 - Elasticsearch
 - ISO8601 or Unix timestamped data
-- Python 2.7
+- Python 3.6
 - pip, see requirements.txt
 - Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev