From 0fd1f524f34a14d29a58b4a36db5f9660aeec176 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 11 Jan 2019 18:34:06 -0500 Subject: [PATCH 001/128] corrected spike aggregation support for avg,min,max --- elastalert/ruletypes.py | 53 ++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index e25ef148..9164379b 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -350,6 +350,20 @@ def mean(self): else: return None + def min(self): + """ The minimum of the value_field in the window. """ + if len(self.data) > 0: + return min([x[1] for x in self.data]) + else: + return None + + def max(self): + """ The maximum of the value_field in the window. """ + if len(self.data) > 0: + return max([x[1] for x in self.data]) + else: + return None + def __iter__(self): return iter(self.data) @@ -421,17 +435,34 @@ def add_data(self, data): if qk is None: qk = 'other' if self.field_value is not None: - count = lookup_es_key(event, self.field_value) - if count is not None: - try: - count = int(count) - except ValueError: - elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) - else: - self.handle_event(event, count, qk) + if self.field_value in event: + count = lookup_es_key(event, self.field_value) + if count is not None: + try: + count = int(count) + except ValueError: + elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) + else: + self.handle_event(event, count, qk) else: self.handle_event(event, 1, qk) + def get_spike_values(self, qk): + """ + extending ref/cur value retrieval logic for spike aggregations + """ + spike_check_type = self.rules.get('metric_agg_type') + if spike_check_type in [None, 'sum', 'value_count']: + # default count logic is appropriate in all these cases + return self.ref_windows[qk].count(), self.cur_windows[qk].count() + elif spike_check_type == 'avg': + return self.ref_windows[qk].mean(), self.cur_windows[qk].mean() + elif spike_check_type == 'min': + return self.ref_windows[qk].min(), self.cur_windows[qk].min() + elif spike_check_type == 'max': + return self.ref_windows[qk].max(), self.cur_windows[qk].max() + + def clear_windows(self, qk, event): # Reset the state and prevent alerts until windows filled again self.ref_windows[qk].clear() @@ -469,7 +500,8 @@ def handle_event(self, event, count, qk='all'): self.add_match(match, qk) self.clear_windows(qk, match) else: - if self.find_matches(self.ref_windows[qk].count(), self.cur_windows[qk].count()): + ref, cur = self.get_spike_values(qk) + if self.find_matches(ref, cur): # skip over placeholder events which have count=0 for match, count in self.cur_windows[qk].data: if count: @@ -481,8 +513,7 @@ def handle_event(self, event, count, qk='all'): def add_match(self, match, qk): extra_info = {} if self.field_value is None: - spike_count = self.cur_windows[qk].count() - reference_count = self.ref_windows[qk].count() + spike_count, reference_count = self.get_spike_values(qk) else: spike_count = self.cur_windows[qk].mean() reference_count = self.ref_windows[qk].mean() From 49d9b053fc9d2f5e7f8548f7533c145a5127283f Mon Sep 17 00:00:00 2001 From: dylanjf Date: Mon, 14 Jan 2019 11:37:23 -0500 Subject: [PATCH 002/128] correct order of variables for spike alert text --- elastalert/ruletypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 9164379b..ca508614 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -513,7 +513,7 @@ def handle_event(self, event, count, qk='all'): def add_match(self, match, qk): extra_info = {} if self.field_value is None: - spike_count, reference_count = self.get_spike_values(qk) + reference_count, spike_count = self.get_spike_values(qk) else: spike_count = self.cur_windows[qk].mean() reference_count = self.ref_windows[qk].mean() From 6849cda20e741ee67c0ceabc9d8aaeaa7fc71377 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Mon, 14 Jan 2019 11:39:05 -0500 Subject: [PATCH 003/128] linebreak fix for travis CI --- elastalert/ruletypes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index ca508614..c1a526d1 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -462,7 +462,6 @@ def get_spike_values(self, qk): elif spike_check_type == 'max': return self.ref_windows[qk].max(), self.cur_windows[qk].max() - def clear_windows(self, qk, event): # Reset the state and prevent alerts until windows filled again self.ref_windows[qk].clear() From ebd3f9eae4ffd1d847f4befa8134d7eef01871fc Mon Sep 17 00:00:00 2001 From: dylanjf Date: Tue, 23 Apr 2019 13:15:05 -0400 Subject: [PATCH 004/128] force elasticsearch 6.3 on build --- elastalert/alerts.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 27e03ac9..fe3bd1e2 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1118,7 +1118,7 @@ def __init__(self, rule): self.slack_channel_override = [self.slack_channel_override] self.slack_title_link = self.rule.get('slack_title_link', '') self.slack_title = self.rule.get('slack_title', '') - self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:') + self.slack_emoji_override = self.rule.get('slack_emoji_override', ':dylan:') self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '') self.slack_msg_color = self.rule.get('slack_msg_color', 'danger') self.slack_parse_override = self.rule.get('slack_parse_override', 'none') diff --git a/requirements.txt b/requirements.txt index 4f23f2ec..5e0c4e47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ blist>=1.3.6 boto3>=1.4.4 configparser>=3.5.0 croniter>=0.3.16 -elasticsearch +elasticsearch==6.3.1 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 From 885896c55216980d013b7061444b60351ec2408a Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Mon, 19 Aug 2019 15:37:14 +0530 Subject: [PATCH 005/128] Add Docs: Elasticsearch Security Privileges Add new page in documentation specifying the Elasticsearch Security Privileges required for ElastAlert --- .../elasticsearch_security_privileges.rst | 35 +++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 36 insertions(+) create mode 100644 docs/source/elasticsearch_security_privileges.rst diff --git a/docs/source/elasticsearch_security_privileges.rst b/docs/source/elasticsearch_security_privileges.rst new file mode 100644 index 00000000..d74497f7 --- /dev/null +++ b/docs/source/elasticsearch_security_privileges.rst @@ -0,0 +1,35 @@ +Elasticsearch Security Privileges +********************************* + +While ElastAlert will just work out-of-the-box for unsecured Elasticsearch, it will need a user with a certain set of permissions to work on secure Elasticseach that allow it to read the documents, check the cluster status etc. + +SearchGuard Permissions +======================= + +The permissions in Elasticsearch are specific to the plugin being used for RBAC. However, the permissions mentioned here can be mapped easily to different plugins other than Searchguard. + +Details about SearchGuard Action Groups: https://docs.search-guard.com/latest/action-groups + + +Writeback Permissions +--------------------------- + +For the global config (which writes to the writeback index), you would need to give all permissions on the writeback indices. +In addition, some permissions related to Cluster Monitor Access are required. + +``Cluster Permissions``: CLUSTER_MONITOR, indices:data/read/scroll* + +``Index Permissions`` (Over Writeback Indices): INDICES_ALL + + +Per Rule Permissions +-------------------------- + +For per rule Elasticsearch config, you would need at least the read permissions on the index you want to query. +Detailed SearchGuard Permissions: + +``Cluster Permissions``: CLUSTER_COMPOSITE_OPS_RO + +``Index Permissions`` (Over the index the rule is querying on): READ, indices:data/read/scroll* + + diff --git a/docs/source/index.rst b/docs/source/index.rst index 4219bf13..cf6d439c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,6 +15,7 @@ Contents: running_elastalert ruletypes elastalert_status + elasticsearch_security_privileges recipes/adding_rules recipes/adding_alerts recipes/writing_filters From 5326ac947b91b17ea43c74a6650344559dbda437 Mon Sep 17 00:00:00 2001 From: Flavio Pompermaier Date: Wed, 11 Dec 2019 10:08:28 +0100 Subject: [PATCH 006/128] Enabling to embed images into email alerter You just need to specify the images dir and the mapping between images keys and values. Example: assets_dir: "/opt/elastalert/email_images" email_image_keys: ["img1"] email_image_values: ["my_logo.png"] alert_text_args: - "winlog.event_data.TargetDomainName" - "winlog.event_data.TargetUserName" - "starttime" alert_text_args: - "winlog.event_data.TargetDomainName" - "winlog.event_data.TargetUserName" - "starttime" alert_text: |

L'utente {0}\{1} ha superato il numero massimo di tentativi di login al minuto (check-time: {2}).



--- elastalert/alerts.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f5ca2207..9d3dfe87 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -11,6 +11,8 @@ import uuid import warnings from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.image import MIMEImage from email.utils import formatdate from html.parser import HTMLParser from smtplib import SMTP @@ -410,6 +412,8 @@ class EmailAlerter(Alerter): def __init__(self, *args): super(EmailAlerter, self).__init__(*args) + self.assets_dir = self.rule.get('assets_dir', '/tmp') + self.images_dictionary = dict(zip( self.rule.get('email_image_keys', []), self.rule.get('email_image_values', []))); self.smtp_host = self.rule.get('smtp_host', 'localhost') self.smtp_ssl = self.rule.get('smtp_ssl', False) self.from_addr = self.rule.get('from_addr', 'ElastAlert') @@ -454,7 +458,17 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - email_msg = MIMEText(body, 'html', _charset='UTF-8') + #email_msg = MIMEText(body, 'html', _charset='UTF-8') # old way + email_msg = MIMEMultipart() + msgText = MIMEText(body, 'html', _charset='UTF-8') + email_msg.attach(msgText) # Added, and edited the previous line + + for image_key in self.images_dictionary: + fp = open(os.path.join(self.assets_dir, self.images_dictionary[image_key]), 'rb') + img = MIMEImage(fp.read()) + fp.close() + img.add_header('Content-ID', '<{}>'.format(image_key)) + email_msg.attach(img) else: email_msg = MIMEText(body, _charset='UTF-8') email_msg['Subject'] = self.create_title(matches) From fbce0fb0ef9a65f57d15188f6d28f33be03fac22 Mon Sep 17 00:00:00 2001 From: Moises Cruz Date: Tue, 21 Apr 2020 20:07:44 +0200 Subject: [PATCH 007/128] agg_type percentiles fixes 2713 --- elastalert/ruletypes.py | 36 +++++++++++++++++++++++++++++------- elastalert/schema.yaml | 6 ++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82..85c08a4a 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1026,6 +1026,7 @@ class MetricAggregationRule(BaseAggregationRule): """ A rule that matches when there is a low number of events given a timeframe. """ required_options = frozenset(['metric_agg_key', 'metric_agg_type']) allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) + allowed_percent_aggregations = frozenset(['percentiles']) def __init__(self, *args): super(MetricAggregationRule, self).__init__(*args) @@ -1035,8 +1036,10 @@ def __init__(self, *args): self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] - if not self.rules['metric_agg_type'] in self.allowed_aggregations: + if not self.rules['metric_agg_type'] in self.allowed_aggregations.union(self.allowed_percent_aggregations): raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations and self.rules['percentile_range'] is None: + raise EAException("percentile_range must be specified for percentiles aggregation") self.rules['aggregation_query_element'] = self.generate_aggregation_query() @@ -1051,14 +1054,20 @@ def get_match_str(self, match): return message def generate_aggregation_query(self): - return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + query = {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: + query[self.metric_key][self.rules['metric_agg_type']]['percents'] = [self.rules['percentile_range']] + return query def check_matches(self, timestamp, query_key, aggregation_data): if "compound_query_key" in self.rules: self.check_matches_recursive(timestamp, query_key, aggregation_data, self.rules['compound_query_key'], dict()) else: - metric_val = aggregation_data[self.metric_key]['value'] + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: + metric_val = list(aggregation_data[self.metric_key]['values'].values())[0] + else: + metric_val = aggregation_data[self.metric_key]['value'] if self.crossed_thresholds(metric_val): match = {self.rules['timestamp_field']: timestamp, self.metric_key: metric_val} @@ -1106,6 +1115,7 @@ class SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule): """ A rule that matches when there is a spike in an aggregated event compared to its reference point """ required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'spike_height', 'spike_type']) allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) + allowed_percent_aggregations = frozenset(['percentiles']) def __init__(self, *args): # We inherit everything from BaseAggregation and Spike, overwrite only what we need in functions below @@ -1113,8 +1123,11 @@ def __init__(self, *args): # MetricAgg alert things self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] - if not self.rules['metric_agg_type'] in self.allowed_aggregations: + + if not self.rules['metric_agg_type'] in self.allowed_aggregations.union(self.allowed_percent_aggregations): raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations and self.rules['percentile_range'] is None: + raise EAException("percentile_range must be specified for percentiles aggregation") # Disabling bucket intervals (doesn't make sense in context of spike to split up your time period) if self.rules.get('bucket_interval'): @@ -1126,7 +1139,10 @@ def generate_aggregation_query(self): """Lifted from MetricAggregationRule, added support for scripted fields""" if self.rules.get('metric_agg_script'): return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} - return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + query = {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: + query[self.metric_key][self.rules['metric_agg_type']]['percents'] = [self.rules['percentile_range']] + return query def add_aggregation_data(self, payload): """ @@ -1140,7 +1156,10 @@ def add_aggregation_data(self, payload): else: # no time / term split, just focus on the agg event = {self.ts_field: timestamp} - agg_value = payload_data[self.metric_key]['value'] + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: + agg_value = list(payload_data[self.metric_key]['values'].values())[0] + else: + agg_value = payload_data[self.metric_key]['value'] self.handle_event(event, agg_value, 'all') return @@ -1160,7 +1179,10 @@ def unwrap_term_buckets(self, timestamp, term_buckets, qk=[]): continue qk_str = ','.join(qk) - agg_value = term_data[self.metric_key]['value'] + if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: + agg_value = list(term_data[self.metric_key]['values'].values())[0] + else: + agg_value = term_data[self.metric_key]['value'] event = {self.ts_field: timestamp, self.rules['query_key']: qk_str} # pass to SpikeRule's tracker diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index cc5d5239..a5b00a5f 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -110,7 +110,7 @@ oneOf: type: {enum: [spike_aggregation]} spike_height: {type: number} spike_type: {enum: ["up", "down", "both"]} - metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count"]} + metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count", "percentiles"]} timeframe: *timeframe use_count_query: {type: boolean} doc_type: {type: string} @@ -120,6 +120,7 @@ oneOf: threshold_ref: {type: number} threshold_cur: {type: number} min_doc_count: {type: integer} + percentile_range: {type: integer} - title: Flatline required: [threshold, timeframe] @@ -153,8 +154,9 @@ oneOf: required: [metric_agg_key,metric_agg_type] properties: type: {enum: [metric_aggregation]} - metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count"]} + metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count", "percentiles"]} #timeframe: *timeframe + percentile_range: {type: integer} - title: Percentage Match required: [match_bucket_filter] From b9963d9e1bd2498bdf76fd2f10e7e71a0181e468 Mon Sep 17 00:00:00 2001 From: Minogiannis Grigoris Date: Fri, 24 Apr 2020 20:17:10 +0300 Subject: [PATCH 008/128] Adjusting elastalert/ruletypes.py so that the functions 'append' and 'append_middle' take into account the scenario whereby an event is None --- elastalert/ruletypes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82..1ccb85d8 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -317,6 +317,8 @@ def append(self, event): """ Add an event to the window. Event should be of the form (dict, count). This will also pop the oldest events and call onRemoved on them until the window size is less than timeframe. """ + if not event or not event[1]: + return self self.data.add(event) self.running_count += event[1] @@ -357,6 +359,8 @@ def __iter__(self): def append_middle(self, event): """ Attempt to place the event in the correct location in our deque. Returns True if successful, otherwise False. """ + if not event or not event[1]: + return self rotation = 0 ts = self.get_ts(event) From 50bcd948460438dc111b309e231a340248b5701d Mon Sep 17 00:00:00 2001 From: Minogiannis Grigoris Date: Fri, 24 Apr 2020 20:47:38 +0300 Subject: [PATCH 009/128] Adjusting elastalert/ruletypes.py so that the functions 'append' and 'append_middle' take into account the scenario whereby an event is None --- elastalert/ruletypes.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 1ccb85d8..00ac4c62 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -317,15 +317,15 @@ def append(self, event): """ Add an event to the window. Event should be of the form (dict, count). This will also pop the oldest events and call onRemoved on them until the window size is less than timeframe. """ - if not event or not event[1]: - return self self.data.add(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] while self.duration() >= self.timeframe: oldest = self.data[0] self.data.remove(oldest) - self.running_count -= oldest[1] + if oldest and oldest[1]: + self.running_count -= oldest[1] self.onRemoved and self.onRemoved(oldest) def duration(self): @@ -359,15 +359,14 @@ def __iter__(self): def append_middle(self, event): """ Attempt to place the event in the correct location in our deque. Returns True if successful, otherwise False. """ - if not event or not event[1]: - return self rotation = 0 ts = self.get_ts(event) # Append left if ts is earlier than first event if self.get_ts(self.data[0]) > ts: self.data.appendleft(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] return # Rotate window until we can insert event @@ -378,7 +377,8 @@ def append_middle(self, event): # This should never happen return self.data.append(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] self.data.rotate(-rotation) From cde4ac325c0991619b46f03be9f400104b0a6a85 Mon Sep 17 00:00:00 2001 From: Ryan Goggin Date: Mon, 27 Apr 2020 15:36:02 -0400 Subject: [PATCH 010/128] Add ca certs and ignore ssl to HTTP Post Added options: - http_post_ca_certs - http_post_ignore_ssl_errors --- elastalert/alerts.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index a453081f..9bd88268 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1956,6 +1956,8 @@ def __init__(self, rule): self.post_static_payload = self.rule.get('http_post_static_payload', {}) self.post_all_values = self.rule.get('http_post_all_values', not self.post_payload) self.post_http_headers = self.rule.get('http_post_headers', {}) + self.post_ca_certs = self.rule.get('http_post_ca_certs') + self.post_ignore_ssl_errors = self.rule.get('http_post_ignore_ssl_errors', False) self.timeout = self.rule.get('http_post_timeout', 10) def alert(self, matches): @@ -1969,12 +1971,18 @@ def alert(self, matches): "Content-Type": "application/json", "Accept": "application/json;charset=utf-8" } + if self.post_ca_certs: + verify = self.post_ca_certs + else: + verify = not self.post_ignore_ssl_errors + headers.update(self.post_http_headers) proxies = {'https': self.post_proxy} if self.post_proxy else None for url in self.post_url: try: response = requests.post(url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, proxies=proxies, timeout=self.timeout) + headers=headers, proxies=proxies, timeout=self.timeout, + verify=verify) response.raise_for_status() except RequestException as e: raise EAException("Error posting HTTP Post alert: %s" % e) From ff899b93f6f75f014871f18af253f2c3c7bc631d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 5 May 2020 16:24:43 +0900 Subject: [PATCH 011/128] Update Docs for Zabbix --- docs/source/ruletypes.rst | 2 +- elastalert/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b..e2420a36 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2242,4 +2242,4 @@ Required: ``zbx_sender_host``: The address where zabbix server is running. ``zbx_sender_port``: The port where zabbix server is listenning. ``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert. -``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert. +``zbx_key``: This field setup the key in the host that receives the value sent by Elastalert. diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index cc5d5239..3fd14744 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -386,4 +386,4 @@ properties: zbx_sender_host: {type: string} zbx_sender_port: {type: integer} zbx_host: {type: string} - zbx_item: {type: string} + zbx_key: {type: string} From 24442676e48df17451b71fe749878076374cbf06 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 5 May 2020 16:57:34 +0900 Subject: [PATCH 012/128] Add LineNotify to alerts mapping --- elastalert/loaders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 77119476..08f06f64 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,7 +77,8 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, - 'hivealerter': alerts.HiveAlerter + 'hivealerter': alerts.HiveAlerter, + 'linenotify': alerts.LineNotifyAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list From f6f57a5bbfae8306fcd29f3b78f382f31038121f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 10 May 2020 06:47:49 +0900 Subject: [PATCH 013/128] Fix SNS Program & Docs --- docs/source/ruletypes.rst | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b..a198abe6 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1670,13 +1670,42 @@ SNS requires one option: Optional: -``aws_access_key``: An access key to connect to SNS with. +``sns_aws_access_key_id``: An access key to connect to SNS with. -``aws_secret_key``: The secret key associated with the access key. +``sns_aws_secret_access_key``: The secret key associated with the access key. -``aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 +``sns_aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 -``profile``: The AWS profile to use. If none specified, the default will be used. +``sns_aws_profile``: The AWS profile to use. If none specified, the default will be used. + +Example When not using aws_profile usage:: + + alert: + - sns + sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' + sns_aws_access_key_id: 'XXXXXXXXXXXXXXXXXX'' + sns_aws_secret_access_key: 'YYYYYYYYYYYYYYYYYYYY' + sns_aws_region: 'us-east-1' # You must nest aws_region within your alert configuration so it is not used to sign AWS requests. + +Example When to use aws_profile usage:: + + # Create ~/.aws/credentials + + [default] + aws_access_key_id = xxxxxxxxxxxxxxxxxxxx + aws_secret_access_key = yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy + + # Create ~/.aws/config + + [default] + region = us-east-1 + + # alert rule setting + + alert: + - sns + sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' + sns_aws_profile: 'default' HipChat ~~~~~~~ From c77c9951edbea7386b82f50f6b0c515b48bace7a Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 10 May 2020 06:50:51 +0900 Subject: [PATCH 014/128] Fix SNS Program & Docs_2 --- elastalert/alerts.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index a453081f..2a04b413 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -935,11 +935,11 @@ class SnsAlerter(Alerter): def __init__(self, *args): super(SnsAlerter, self).__init__(*args) self.sns_topic_arn = self.rule.get('sns_topic_arn', '') - self.aws_access_key_id = self.rule.get('aws_access_key_id') - self.aws_secret_access_key = self.rule.get('aws_secret_access_key') - self.aws_region = self.rule.get('aws_region', 'us-east-1') + self.sns_aws_access_key_id = self.rule.get('sns_aws_access_key_id') + self.sns_aws_secret_access_key = self.rule.get('sns_aws_secret_access_key') + self.sns_aws_region = self.rule.get('sns_aws_region', 'us-east-1') self.profile = self.rule.get('boto_profile', None) # Deprecated - self.profile = self.rule.get('aws_profile', None) + self.profile = self.rule.get('sns_aws_profile', None) def create_default_title(self, matches): subject = 'ElastAlert: %s' % (self.rule['name']) @@ -948,12 +948,15 @@ def create_default_title(self, matches): def alert(self, matches): body = self.create_alert_body(matches) - session = boto3.Session( - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - region_name=self.aws_region, - profile_name=self.profile - ) + if self.profile is None: + session = boto3.Session( + aws_access_key_id=self.sns_aws_access_key_id, + aws_secret_access_key=self.sns_aws_access_key_id, + region_name=self.sns_aws_region + ) + else: + session = boto3.Session(profile_name=self.profile) + sns_client = session.client('sns') sns_client.publish( TopicArn=self.sns_topic_arn, From 72709790c3dff1748467e108a44bb92dc1bcdb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jasper=20J=C3=BCrgensen?= Date: Sun, 10 May 2020 17:25:28 +0200 Subject: [PATCH 015/128] Adds --silence_qk_value option to elastalert This enables the user to silence a rule only for a specific query_key value and not only the whole rule. See Issue #2777 --- elastalert/elastalert.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 24b10ced..fd1eeb5f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -83,6 +83,11 @@ def parse_args(self, args): parser.add_argument('--rule', dest='rule', help='Run only a specific rule (by filename, must still be in rules folder)') parser.add_argument('--silence', dest='silence', help='Silence rule for a time period. Must be used with --rule. Usage: ' '--silence =, eg. --silence hours=2') + parser.add_argument( + "--silence_qk_value", + dest="silence_qk_value", + help="Silence the rule only for this specific query key value.", + ) parser.add_argument('--start', dest='start', help='YYYY-MM-DDTHH:MM:SS Start querying from this timestamp. ' 'Use "NOW" to start from current time. (Default: present)') parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present)') @@ -1853,7 +1858,10 @@ def silence(self, silence_cache_key=None): # With --rule, self.rules will only contain that specific rule if not silence_cache_key: - silence_cache_key = self.rules[0]['name'] + "._silence" + if self.args.silence_qk_value: + silence_cache_key = self.rules[0]['name'] + "." + self.args.silence_qk_value + else: + silence_cache_key = self.rules[0]['name'] + "._silence" try: silence_ts = parse_deadline(self.args.silence) From 496a04fb89e5e99021bc894537c35a514c2f6414 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 23 May 2020 00:50:32 +0900 Subject: [PATCH 016/128] Sync requirements.txt and setup.py & update py-zabbix --- requirements.txt | 6 +++--- setup.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index c66ca8d7..f5fb0b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,12 +12,12 @@ jira>=1.0.10,<1.0.15 jsonschema>=3.0.2 mock>=2.0.0 prison>=0.1.2 -py-zabbix==1.1.3 +py-zabbix>=1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 python-magic>=0.4.15 PyYAML>=5.1 -requests>=2.0.0 +requests>=2.10.0 stomp.py>=4.1.17 texttable>=0.8.8 -twilio==6.0.0 +twilio>=6.0.0,<6.1 diff --git a/setup.py b/setup.py index 30ef9495..048eb5fa 100644 --- a/setup.py +++ b/setup.py @@ -42,12 +42,13 @@ 'prison>=0.1.2', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', - 'PyYAML>=3.12', + 'PyYAML>=5.1', 'requests>=2.10.0', 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', 'python-magic>=0.4.15', - 'cffi>=1.11.5' + 'cffi>=1.11.5', + 'py-zabbix>=1.1.3' ] ) From f8b0415bff4f78e27ae1cc13502196c7512355bd Mon Sep 17 00:00:00 2001 From: Raghu Date: Wed, 10 Jun 2020 09:17:36 +0530 Subject: [PATCH 017/128] Added Squadcast http post alert config --- docs/source/ruletypes.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b..9f364811 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2164,6 +2164,20 @@ Example usage:: http_post_headers: authorization: Basic 123dr3234 +Squadcast +~~~~~~~~ + +Alerts can be sent to Squadcast using the `http post` method described above and Squadcast will process it and send Phone, SMS, Email and Push notifications to the relevant person(s) and let them take actions. + +Configuration variables in rules YAML file:: + + alert: post + http_post_url: + http_post_static_payload: + Title: + http_post_all_values: true + +For more details, you can refer the `Squadcast documentation `_. Alerter ~~~~~~~ From fa5a16b7808e49489ce6bbe6ac35f65f25b2f4a0 Mon Sep 17 00:00:00 2001 From: Raghu Date: Wed, 10 Jun 2020 09:20:41 +0530 Subject: [PATCH 018/128] Adjusted the space in the YAML config --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 9f364811..fb6ff813 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2174,7 +2174,7 @@ Configuration variables in rules YAML file:: alert: post http_post_url: http_post_static_payload: - Title: + Title: http_post_all_values: true For more details, you can refer the `Squadcast documentation `_. From c15364ab92f71183b987816d1815f615a4c0e8ad Mon Sep 17 00:00:00 2001 From: Raghu Date: Wed, 10 Jun 2020 09:23:00 +0530 Subject: [PATCH 019/128] Replaced tabs with spaces --- docs/source/ruletypes.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index fb6ff813..6ce23ec1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2171,11 +2171,11 @@ Alerts can be sent to Squadcast using the `http post` method described above and Configuration variables in rules YAML file:: - alert: post - http_post_url: - http_post_static_payload: - Title: - http_post_all_values: true + alert: post + http_post_url: + http_post_static_payload: + Title: + http_post_all_values: true For more details, you can refer the `Squadcast documentation `_. From 8a3fa307da188536da8572f3f88734e718aa7bc8 Mon Sep 17 00:00:00 2001 From: Raghu Date: Tue, 16 Jun 2020 10:15:52 +0530 Subject: [PATCH 020/128] Fixed the build error by matching the number of ~ to the length of the heading. --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 6ce23ec1..c493c39a 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2165,7 +2165,7 @@ Example usage:: authorization: Basic 123dr3234 Squadcast -~~~~~~~~ +~~~~~~~~~ Alerts can be sent to Squadcast using the `http post` method described above and Squadcast will process it and send Phone, SMS, Email and Push notifications to the relevant person(s) and let them take actions. From a107491800ffd3e666d2dd570aefa952e197d280 Mon Sep 17 00:00:00 2001 From: Swapnil Suryawanshi Date: Tue, 23 Jun 2020 15:20:45 +0530 Subject: [PATCH 021/128] fix attribute error is raised when query ran for future --- elastalert/elastalert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 24b10ced..f99a3229 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -870,15 +870,16 @@ def run_rule(self, rule, endtime, starttime=None): rule['original_starttime'] = rule['starttime'] rule['scrolling_cycle'] = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 + # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: logging.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) return 0 # Run the rule. If querying over a large time period, split it up into segments - self.thread_data.num_hits = 0 - self.thread_data.num_dupes = 0 - self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] From 6b04f9e12c076bb49a2b37082d779916b1666d4e Mon Sep 17 00:00:00 2001 From: Ryan Goggin Date: Thu, 9 Jul 2020 08:23:23 -0400 Subject: [PATCH 022/128] Fix tests to include new kwarg verify to requests.post --- tests/alerts_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 5cd61ae7..d5ac49ff 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1672,7 +1672,8 @@ def test_http_alerter_with_payload(): data=mock.ANY, headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, - timeout=10 + timeout=10, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1707,7 +1708,8 @@ def test_http_alerter_with_payload_all_values(): data=mock.ANY, headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, - timeout=10 + timeout=10, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1739,7 +1741,8 @@ def test_http_alerter_without_payload(): data=mock.ANY, headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'}, proxies=None, - timeout=10 + timeout=10, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) From a7144d156016830cb383ed96deb622a9128d0fb3 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 25 Jul 2020 13:57:00 +0900 Subject: [PATCH 023/128] Remove Stride integration The Stride service was shutdown on February 15th, 2019. --- docs/source/ruletypes.rst | 20 --- elastalert/alerts.py | 93 ------------- elastalert/loaders.py | 1 - elastalert/schema.yaml | 6 - tests/alerts_test.py | 278 -------------------------------------- 5 files changed, 398 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b..b5ad1f61 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1715,26 +1715,6 @@ If set, it will mention the users, no matter if the original message format is s Valid values: list of strings. Defaults to ``[]``. - -Stride -~~~~~~~ - -Stride alerter will send a notification to a predefined Stride room. The body of the notification is formatted the same as with other alerters. -Simple HTML such as and tags will be parsed into a format that Stride can consume. - -The alerter requires the following two options: - -``stride_access_token``: The randomly generated notification token created by Stride. - -``stride_cloud_id``: The site_id associated with the Stride site you want to send the alert to. - -``stride_conversation_id``: The conversation_id associated with the Stride conversation you want to send the alert to. - -``stride_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false. - -``stride_proxy``: By default ElastAlert will not use a network proxy to send notifications to Stride. Set this option using ``hostname:port`` if you need to use a proxy. - - MS Teams ~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..48c834e8 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1985,99 +1985,6 @@ def get_info(self): 'http_post_webhook_url': self.post_url} -class StrideHTMLParser(HTMLParser): - """Parse html into stride's fabric structure""" - - def __init__(self): - """ - Define a couple markup place holders. - """ - self.content = [] - self.mark = None - HTMLParser.__init__(self) - - def handle_starttag(self, tag, attrs): - """Identify and verify starting tag is fabric compatible.""" - if tag == 'b' or tag == 'strong': - self.mark = dict(type='strong') - if tag == 'u': - self.mark = dict(type='underline') - if tag == 'a': - self.mark = dict(type='link', attrs=dict(attrs)) - - def handle_endtag(self, tag): - """Clear mark on endtag.""" - self.mark = None - - def handle_data(self, data): - """Construct data node for our data.""" - node = dict(type='text', text=data) - if self.mark: - node['marks'] = [self.mark] - self.content.append(node) - - -class StrideAlerter(Alerter): - """ Creates a Stride conversation message for each alert """ - - required_options = frozenset( - ['stride_access_token', 'stride_cloud_id', 'stride_conversation_id']) - - def __init__(self, rule): - super(StrideAlerter, self).__init__(rule) - - self.stride_access_token = self.rule['stride_access_token'] - self.stride_cloud_id = self.rule['stride_cloud_id'] - self.stride_conversation_id = self.rule['stride_conversation_id'] - self.stride_ignore_ssl_errors = self.rule.get('stride_ignore_ssl_errors', False) - self.stride_proxy = self.rule.get('stride_proxy', None) - self.url = 'https://api.atlassian.com/site/%s/conversation/%s/message' % ( - self.stride_cloud_id, self.stride_conversation_id) - - def alert(self, matches): - body = self.create_alert_body(matches).strip() - - # parse body with StrideHTMLParser - parser = StrideHTMLParser() - parser.feed(body) - - # Post to Stride - headers = { - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(self.stride_access_token) - } - - # set https proxy, if it was provided - proxies = {'https': self.stride_proxy} if self.stride_proxy else None - - # build stride json payload - # https://developer.atlassian.com/cloud/stride/apis/document/structure/ - payload = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': parser.content} - ]} - ]}} - - try: - if self.stride_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() - response = requests.post( - self.url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.stride_ignore_ssl_errors, - proxies=proxies) - warnings.resetwarnings() - response.raise_for_status() - except RequestException as e: - raise EAException("Error posting to Stride: %s" % e) - elastalert_logger.info( - "Alert sent to Stride conversation %s" % self.stride_conversation_id) - - def get_info(self): - return {'type': 'stride', - 'stride_cloud_id': self.stride_cloud_id, - 'stride_converstation_id': self.stride_converstation_id} - - class LineNotifyAlerter(Alerter): """ Created a Line Notify for each alert """ required_option = frozenset(["linenotify_access_token"]) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 77119476..053c68ac 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -63,7 +63,6 @@ class RulesLoader(object): 'command': alerts.CommandAlerter, 'sns': alerts.SnsAlerter, 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, 'mattermost': alerts.MattermostAlerter, diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index cc5d5239..7a542686 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -270,12 +270,6 @@ properties: hipchat_from: {type: string} hipchat_mentions: {type: array, items: {type: string}} - ### Stride - stride_access_token: {type: string} - stride_cloud_id: {type: string} - stride_conversation_id: {type: string} - stride_ignore_ssl_errors: {type: boolean} - ### Slack slack_webhook_url: *arrayOfString slack_username_override: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 5cd61ae7..59ee5746 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -20,7 +20,6 @@ from elastalert.alerts import MsTeamsAlerter from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter -from elastalert.alerts import StrideAlerter from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add @@ -2086,283 +2085,6 @@ def test_resolving_rule_references(ea): assert 'the_owner' == alert.rule['nested_dict']['nested_owner'] -def test_stride_plain_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "{0}\n\n@timestamp: {1}\nsomefield: {2}".format( - rule['name'], match['@timestamp'], match['somefield'] - ) - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_underline_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Underline Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Underline Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'underline'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_bold_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Bold Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Bold Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'strong'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_strong_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Bold Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Bold Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'strong'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_hyperlink(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Link', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Link" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'link', 'attrs': {'href': 'http://stride.com'}} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_html(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something. Link', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': 'Alert', 'marks': [ - {'type': 'strong'} - ]}, - {'type': 'text', 'text': ': we found something. '}, - {'type': 'text', 'text': 'Link', 'marks': [ - {'type': 'link', 'attrs': {'href': 'http://stride.com'}} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - def test_hipchat_body_size_limit_text(): rule = { 'name': 'Test Rule', From ef8484c87e2568f4197feda222b19ed144313e92 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 25 Jul 2020 14:12:12 +0900 Subject: [PATCH 024/128] Remove import HTMLParser --- elastalert/alerts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 48c834e8..0130063b 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -12,7 +12,6 @@ import warnings from email.mime.text import MIMEText from email.utils import formatdate -from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError From d2cb0ed42c06f473fa77d7926b273534818da380 Mon Sep 17 00:00:00 2001 From: Flavio Pompermaier Date: Mon, 27 Jul 2020 10:47:36 +0200 Subject: [PATCH 025/128] Fixed checkstyle errors --- elastalert/alerts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 9d3dfe87..7a964a58 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -413,7 +413,7 @@ def __init__(self, *args): super(EmailAlerter, self).__init__(*args) self.assets_dir = self.rule.get('assets_dir', '/tmp') - self.images_dictionary = dict(zip( self.rule.get('email_image_keys', []), self.rule.get('email_image_values', []))); + self.images_dictionary = dict(zip(self.rule.get('email_image_keys', []), self.rule.get('email_image_values', []))) self.smtp_host = self.rule.get('smtp_host', 'localhost') self.smtp_ssl = self.rule.get('smtp_ssl', False) self.from_addr = self.rule.get('from_addr', 'ElastAlert') @@ -458,7 +458,7 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - #email_msg = MIMEText(body, 'html', _charset='UTF-8') # old way + # email_msg = MIMEText(body, 'html', _charset='UTF-8') # old way email_msg = MIMEMultipart() msgText = MIMEText(body, 'html', _charset='UTF-8') email_msg.attach(msgText) # Added, and edited the previous line From b739aaa6d57e2232429c77a15fad4034e3bd6abb Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 13 Aug 2020 01:44:46 +0900 Subject: [PATCH 026/128] kibana discover 7.4/7.5/7.6/7.7/7.8 support --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b..e2ecfb67 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -553,9 +553,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8` -``kibana_discover_version: '7.3'`` +``kibana_discover_version: '7.8'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 7e4dbb5d..3fb25214 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -14,7 +14,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index cc5d5239..5946d9c0 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -217,7 +217,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index f06fe4e0..70bd4377 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From 39a04aa9b038dde24d6a5373fe6000137a927edb Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 22 Aug 2020 21:25:17 +0900 Subject: [PATCH 027/128] Kibana 7.9 --- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 3fb25214..b332facc 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -14,7 +14,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 5946d9c0..ba265a73 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -217,7 +217,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index 70bd4377..ae5c8bca 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.8', + 'kibana_discover_version': '7.9', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.8', + 'kibana_discover_version': '7.9', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.8', + 'kibana_discover_version': '7.9', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.8', + 'kibana_discover_version': '7.9', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.8', + 'kibana_discover_version': '7.9', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From 87ba1d1ffc7d4c36bb2fd6817e5d7f487eaf209b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Sep 2020 06:30:50 -0400 Subject: [PATCH 028/128] TheHive alerter: Allow severity and tlp to be set by rule --- elastalert/alerts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..b881ec2e 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2154,7 +2154,10 @@ def alert(self, matches): n += 1 custom_fields[cf_key] = cf elif isinstance(alert_config_value, str): - alert_config[alert_config_field] = alert_config_value.format(**context) + alert_value = alert_config_value.format(**context) + if alert_config_field in ['severity', 'tlp']: + alert_value = int(alert_value) + alert_config[alert_config_field] = alert_value elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: From 59ad4ac53c838aa76f85671c1252d28787fcc452 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Sat, 12 Sep 2020 06:46:10 +0000 Subject: [PATCH 029/128] fix a configuration options of docs --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ff376371..4426c00a 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -140,7 +140,7 @@ Rule Configuration Cheat Sheet +----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+ | ``ignore_null`` (boolean, no default) | | | Req | Req | | | | | | +----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+ -| ``query_key`` (string, no default) | Opt | | | Req | Opt | Opt | Opt | Req | Opt | +| ``query_key`` (string or list, no default) | Opt | | | Req | Opt | Opt | Opt | Req | Opt | +----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+ | ``aggregation_key`` (string, no default) | Opt | | | | | | | | | +----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+ @@ -160,7 +160,7 @@ Rule Configuration Cheat Sheet | | | | | | | | | | | |``doc_type`` (string, no default) | | | | | | | | | | | | | | | | | | | | | -|``query_key`` (string, no default) | | | | | | | | | | +|``query_key`` (string or list, no default) | | | | | | | | | | | | | | | | | | | | | |``terms_size`` (int, default 50) | | | | | | | | | | +----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+ From 734caf2bb55ea9e4c086d6e1bcacf5dbe8710455 Mon Sep 17 00:00:00 2001 From: Feroz Date: Mon, 5 Oct 2020 11:29:50 +0100 Subject: [PATCH 030/128] Fix the use of timestamps in the Jira ticket titles This has come about because we currently use records which have timestamps like: ``` "@timestamp": { "min": "2020-10-03T07:05:01.987Z", "max": "2020-10-05T07:05:09.572Z" }, ``` These work fine with the `timestamp_field` set to `@timestamp.min`. However, when we try to create a Jira alert using ElastAlert from these, we run into a Python error: ``` File "/usr/local/lib/python3.6/site-packages/elastalert/alerts.py", line 875, in create_default_title, title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time'))), KeyError: '@timestamp.min' ``` This is because matches[0][self.rule['timestamp_field']] attempts to access the `timestamp_field` directly rather than using a `get()`. The proposed fix will not change any existing behaviour, but will skip the addition of a timestamp to the ticket title if the required field doesn't exist, rather than throwing an error and disabling the rule. --- elastalert/alerts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..f2f31853 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -868,7 +868,9 @@ def create_default_title(self, matches, for_search=False): if for_search: return title - title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time'))) + timestamp = matches[0].get(self.rule['timestamp_field']) + if timestamp: + title += ' - %s' % (pretty_ts(timestamp, self.rule.get('use_local_time'))) # Add count for spikes count = matches[0].get('spike_count') From 62924c4b2e7ecf071bb8cde46a726768df7d3ab0 Mon Sep 17 00:00:00 2001 From: Kevin Rose Date: Thu, 1 Oct 2020 17:06:32 -0400 Subject: [PATCH 031/128] Add support for custom_details in the PagerDuty alerter v2 module --- docs/source/ruletypes.rst | 5 +++++ elastalert/alerts.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ff376371..0823c82e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1915,6 +1915,11 @@ See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 ``pagerduty_v2_payload_source_args``: If set, and ``pagerduty_v2_payload_source`` is a formattable string, Elastalert will format the source based on the provided array of fields from the rule or match. +``pagerduty_v2_payload_custom_details``: List of keys:values to use as the content of the custom_details payload. Example - ip:clientip will map the value from the clientip index of Elasticsearch to JSON key named ip. + +``pagerduty_v2_payload_include_all_info``: If True, this will include the entire Elasticsearch document as a custom detail field called "information" in the PagerDuty alert. + + PagerTree ~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..cca59b0d 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1357,6 +1357,8 @@ def __init__(self, rule): self.pagerduty_v2_payload_severity = self.rule.get('pagerduty_v2_payload_severity', 'critical') self.pagerduty_v2_payload_source = self.rule.get('pagerduty_v2_payload_source', 'ElastAlert') self.pagerduty_v2_payload_source_args = self.rule.get('pagerduty_v2_payload_source_args', None) + self.pagerduty_v2_payload_custom_details = self.rule.get('pagerduty_v2_payload_custom_details', {}) + self.pagerduty_v2_payload_include_all_info = self.rule.get('pagerduty_v2_payload_include_all_info', True) if self.pagerduty_api_version == 'v2': self.url = 'https://events.pagerduty.com/v2/enqueue' @@ -1369,6 +1371,13 @@ def alert(self, matches): # post to pagerduty headers = {'content-type': 'application/json'} if self.pagerduty_api_version == 'v2': + + custom_details_payload = {'information': body} if self.pagerduty_v2_payload_include_all_info else {} + if self.pagerduty_v2_payload_custom_details: + for match in matches: + for custom_details_key, es_key in list(self.pagerduty_v2_payload_custom_details.items()): + custom_details_payload[custom_details_key] = lookup_es_key(match, es_key) + payload = { 'routing_key': self.pagerduty_service_key, 'event_action': self.pagerduty_event_type, @@ -1389,9 +1398,7 @@ def alert(self, matches): self.pagerduty_v2_payload_source_args, matches), 'summary': self.create_title(matches), - 'custom_details': { - 'information': body, - }, + 'custom_details': custom_details_payload, }, } match_timestamp = lookup_es_key(matches[0], self.rule.get('timestamp_field', '@timestamp')) From d818bfa81d29731198ca8248811778e6cc85c551 Mon Sep 17 00:00:00 2001 From: Daichi Kimura Date: Tue, 20 Oct 2020 01:16:06 +0900 Subject: [PATCH 032/128] Fix initializing self.thread_data.alerts_sent for running elastalert-test-rule --- elastalert/elastalert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b078c86d..24c9f884 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -159,6 +159,7 @@ def __init__(self, args): self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) + self.thread_data.alerts_sent = 0 self.thread_data.num_hits = 0 self.thread_data.num_dupes = 0 self.scheduler = BackgroundScheduler() From 648f1996d47204167626d5763a6cc8e0ec44780a Mon Sep 17 00:00:00 2001 From: Sherif Abdel-Naby Date: Wed, 21 Oct 2020 06:07:38 +0200 Subject: [PATCH 033/128] Add Jinja Requirement to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2845836a..e2906230 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=2.0.0', + 'Jinja2==2.10.1', 'jsonschema>=3.0.2', 'mock>=2.0.0', 'prison>=0.1.2', From 1b4875b26d32f70cc392e5d3cb6e2917a39df329 Mon Sep 17 00:00:00 2001 From: Sherif Abdel-Naby Date: Sat, 17 Oct 2020 20:09:25 +0200 Subject: [PATCH 034/128] Add Jinja2 Template option to `alert_text_type` --- elastalert/alerts.py | 10 ++++++++-- elastalert/loaders.py | 6 ++++++ elastalert/schema.yaml | 2 +- requirements.txt | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..8dacfcfc 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -63,7 +63,13 @@ def _ensure_new_line(self): def _add_custom_alert_text(self): missing = self.rule.get('alert_missing_value', '') alert_text = str(self.rule.get('alert_text', '')) - if 'alert_text_args' in self.rule: + if 'alert_text_jinja' == self.rule.get('alert_text_type'): + # Top fields are accessible via `{{field_name}}` or `{{jinja_root_name['field_name']}}` + # `jinja_root_name` dict is useful when accessing *fields with dots in their keys*, + # as Jinja treat dot as a nested field. + alert_text = self.rule.get("jinja_template").render(**self.match, + **{self.rule['jinja_root_name']: self.match}) + elif 'alert_text_args' in self.rule: alert_text_args = self.rule.get('alert_text_args') alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args] @@ -142,7 +148,7 @@ def __str__(self): self._add_custom_alert_text() self._ensure_new_line() - if self.rule.get('alert_text_type') != 'alert_text_only': + if self.rule.get('alert_text_type') != 'alert_text_only' and self.rule.get('alert_text_type') != 'alert_text_jinja': self._add_rule_text() self._ensure_new_line() if self.rule.get('top_count_keys'): diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 77119476..5729c4ed 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -9,6 +9,7 @@ import jsonschema import yaml import yaml.scanner +from jinja2 import Template from staticconf.loader import yaml_loader from . import alerts @@ -279,6 +280,7 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('_source_enabled', True) rule.setdefault('use_local_time', True) rule.setdefault('description', "") + rule.setdefault('jinja_root_name', "_data") # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() @@ -401,6 +403,10 @@ def _dt_to_ts_with_format(dt): if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') + # Compile Jinja Template + if rule.get('alert_text_type') == 'alert_text_jinja': + rule["jinja_template"] = Template(str(rule.get('alert_text', ''))) + def load_modules(self, rule, args=None): """ Loads things that could be modules. Enhancements, alerts and rule type. """ # Set match enhancements diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 1241315d..852702a8 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -227,7 +227,7 @@ properties: alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} alert_text_kw: {type: object} - alert_text_type: {enum: [alert_text_only, exclude_fields, aggregation_summary_only]} + alert_text_type: {enum: [alert_text_only, alert_text_jinja, exclude_fields, aggregation_summary_only]} alert_missing_value: {type: string} timestamp_field: {type: string} field: {} diff --git a/requirements.txt b/requirements.txt index 9c32052d..089a1112 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ croniter>=0.3.16 elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 +Jinja2==2.10.1 jira>=1.0.10,<1.0.15 jsonschema>=3.0.2 mock>=2.0.0 From 8f0b5d974baa56eb5ff5a4acfc95fed8982b746d Mon Sep 17 00:00:00 2001 From: Sherif Abdel-Naby Date: Sat, 17 Oct 2020 20:16:14 +0200 Subject: [PATCH 035/128] Update Docs about using Jinja2 Template in `alert_text`. --- README.md | 19 ++++++++++++++++++- docs/source/elastalert.rst | 2 ++ docs/source/ruletypes.rst | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 99acc02e..ca626a9f 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,24 @@ The data for when an alert will fire again is stored in Elasticsearch in the ``e ### How can I change what's in the alert? -You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only``, it will be the entirety of the alert. You can also add different fields from the alert by using Python style string formatting and ``alert_text_args``. For example +You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: + +With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. + +``` +alert_text_type: alert_text_jinja + +alert_text: | + Alert triggered! *({{num_hits}} Matches!)* + Something happened with {{username}} ({{email}}) + {{description|truncate}} + +``` + +> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. +> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. + +With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example ``` alert_text: "Something happened with {0} at {1}" diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index b1008c3c..f7fc59c2 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -203,6 +203,8 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``skip_invalid``: If ``True``, skip invalid files instead of exiting. +``jinja_root_name``: When using a Jinja template, specify the name of the root field name in the template. The default is ``_data``. + Logging ------- diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ff376371..d961cb26 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1361,9 +1361,30 @@ There are several ways to format the body text of the various types of events. I top_counts = top_counts_header, LF, top_counts_value field_values = Field, ": ", Value -Similarly to ``alert_subject``, ``alert_text`` can be further formatted using standard Python formatting syntax. +Similarly to ``alert_subject``, ``alert_text`` can be further formatted using Jinja2 Templates or Standard Python Formatting Syntax + +1. Jinja Template + +By setting ``alert_text_type: alert_text_jinja`` you can use jinja2 templates in ``alert_text``. :: + + alert_text_type: alert_text_jinja + + alert_text: | + Alert triggered! *({{num_hits}} Matches!)* + Something happened with {{username}} ({{email}}) + {{description|truncate}} + +Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. +If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. + +2. Standard Python Formatting Syntax + The field names whose values will be used as the arguments can be passed with ``alert_text_args`` or ``alert_text_kw``. -You may also refer to any top-level rule property in the ``alert_subject_args``, ``alert_text_args``, ``alert_missing_value``, and ``alert_text_kw fields``. However, if the matched document has a key with the same name, that will take preference over the rule property. +You may also refer to any top-level rule property in the ``alert_subject_args``, ``alert_text_args``, ``alert_missing_value``, and ``alert_text_kw fields``. However, if the matched document has a key with the same name, that will take preference over the rule property. :: + + alert_text: "Something happened with {0} at {1}" + alert_text_type: alert_text_only + alert_text_args: ["username", "@timestamp"] By default:: @@ -1383,6 +1404,14 @@ With ``alert_text_type: alert_text_only``:: alert_text + +With ``alert_text_type: alert_text_jinja``:: + + body = rule_name + + alert_text + + With ``alert_text_type: exclude_fields``:: body = rule_name @@ -1501,7 +1530,7 @@ by the smtp server. ``bcc``: This adds the BCC emails to the list of recipients but does not show up in the email message. By default, this is left empty. ``email_format``: If set to ``html``, the email's MIME type will be set to HTML, and HTML content should correctly render. If you use this, -you need to put your own HTML into ``alert_text`` and use ``alert_text_type: alert_text_only``. +you need to put your own HTML into ``alert_text`` and use ``alert_text_type: alert_text_jinja`` Or ``alert_text_type: alert_text_only``. Jira ~~~~ From 6388d6768dd6942aaf07925b25491e9db14a790e Mon Sep 17 00:00:00 2001 From: Cheng Liangyu Date: Thu, 22 Oct 2020 21:47:40 +0800 Subject: [PATCH 036/128] Setting size to 0 avoids executing the fetch phase of the search making the request more efficient https://www.elastic.co/guide/en/elasticsearch/reference/current/returning-only-agg-results.html --- elastalert/ruletypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82..740e74ea 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -674,7 +674,7 @@ def get_all_terms(self, args): time_filter = {self.rules['timestamp_field']: {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)}} query_template['filter'] = {'bool': {'must': [{'range': time_filter}]}} - query = {'aggs': {'filtered': query_template}} + query = {'aggs': {'filtered': query_template}, 'size': 0} if 'filter' in self.rules: for item in self.rules['filter']: From a9f0d1d8488b02e9e2d96a65d49d70fc12733f0a Mon Sep 17 00:00:00 2001 From: Dennis Boone Date: Wed, 28 Oct 2020 15:19:49 -0400 Subject: [PATCH 037/128] Fix for the mapping error reported in #2899. --- elastalert/es_mappings/6/elastalert.json | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index 645a6776..2cc97bcf 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -29,6 +29,7 @@ "format": "dateOptionalTime" }, "match_body": { + "enabled": "false", "type": "object" }, "aggregate_id": { From 873f41bac13ea6a2264121c2bcd2e047ecb1a4e5 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Wed, 4 Nov 2020 22:44:39 +0900 Subject: [PATCH 038/128] Fix Stomp --- elastalert/alerts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..d7d8cfce 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -371,7 +371,6 @@ def alert(self, matches): conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)], use_ssl=self.stomp_ssl) - conn.start() conn.connect(self.stomp_login, self.stomp_password) # Ensures that the CONNECTED frame is received otherwise, the disconnect call will fail. time.sleep(1) From 66d5d224801793c18e981123363f1237d9f35d1d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 7 Nov 2020 19:31:26 +0900 Subject: [PATCH 039/128] Del the forgotten code of new_style_string_format --- elastalert/alerts.py | 4 ---- tests/alerts_test.py | 26 -------------------------- 2 files changed, 30 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518..fe261b5a 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -896,10 +896,6 @@ def __init__(self, *args): logging.warning('Warning! You could be vulnerable to shell injection!') self.rule['command'] = [self.rule['command']] - self.new_style_string_format = False - if 'new_style_string_format' in self.rule and self.rule['new_style_string_format']: - self.new_style_string_format = True - def alert(self, matches): # Format the command and arguments try: diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 5cd61ae7..894de728 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1072,32 +1072,6 @@ def test_command(): alert.alert([match]) assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True) - # Test command as string with formatted arg (new-style string format) - rule = {'command': '/bin/test/ --arg {match[somefield]}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False) - - rule = {'command': '/bin/test/ --arg {match[nested][field]}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test --arg 1', stdin=subprocess.PIPE, shell=False) - - # Test command as string without formatted arg (new-style string format) - rule = {'command': '/bin/test/foo.sh', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True) - - rule = {'command': '/bin/test/foo.sh {{bar}}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test/foo.sh {bar}', stdin=subprocess.PIPE, shell=True) - # Test command with pipe_match_json rule = {'command': ['/bin/test/', '--arg', '%(somefield)s'], 'pipe_match_json': True} From 814b415de4381e07aed1d148a7ad6d1a9e1e1a62 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 7 Nov 2020 21:44:11 +0900 Subject: [PATCH 040/128] document SNS to AWS SNS --- docs/source/elastalert.rst | 2 +- docs/source/ruletypes.rst | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index b1008c3c..3bd400d0 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -35,7 +35,7 @@ Currently, we have support built in for these alert types: - Email - JIRA - OpsGenie -- SNS +- AWS SNS - HipChat - Slack - Telegram diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a198abe6..3fb818a4 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1657,14 +1657,14 @@ Example usage:: Environment: '$VAR' # environment variable Message: { field: message } # field in the first match -SNS -~~~ +AWS SNS +~~~~~~~ -The SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. -The SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or +The AWS SNS alerter will send an AWS SNS notification. The body of the notification is formatted the same as with other alerters. +The AWS SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or via environment variables. See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for details. -SNS requires one option: +AWS SNS requires one option: ``sns_topic_arn``: The SNS topic's ARN. For example, ``arn:aws:sns:us-east-1:123456789:somesnstopic`` From c62c1a732b85deb295cd7ac2f9d87fbecf2e5719 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 22 Nov 2020 20:42:26 +0900 Subject: [PATCH 041/128] Fix elasticsearch-py versionup test_rule.py error --- elastalert/test_rule.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 06100aa0..af1eaa49 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -83,7 +83,7 @@ def test_file(self, conf, args): # Get one document for schema try: - res = es_client.search(index, size=1, body=query, ignore_unavailable=True) + res = es_client.search(index=index, size=1, body=query, ignore_unavailable=True) except Exception as e: print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -109,7 +109,7 @@ def test_file(self, conf, args): five=conf['five'] ) try: - res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True) + res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True) except Exception as e: print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -153,7 +153,7 @@ def test_file(self, conf, args): # Download up to max_query_size (defaults to 10,000) documents to save if (args.save or args.formatted_output) and not args.count: try: - res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True) + res = es_client.search(index=index, size=args.max_query_size, body=query, ignore_unavailable=True) except Exception as e: print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) From af6e71affd1caad8385ee35b2c9806ddf310c607 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Nov 2020 08:25:17 -0500 Subject: [PATCH 042/128] Switch out abandoned blist lib for sortedcontainers lib --- elastalert/ruletypes.py | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82..7a889e80 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -3,7 +3,7 @@ import datetime import sys -from blist import sortedlist +from sortedcontainers import SortedKeyList as sortedlist from .util import add_raw_postfix from .util import dt_to_ts diff --git a/requirements.txt b/requirements.txt index c66ca8d7..6f6d6f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ apscheduler>=3.3.0 aws-requests-auth>=0.3.0 -blist>=1.3.6 +sortedcontainers>=2.2.2 boto3>=1.4.4 cffi>=1.11.5 configparser>=3.5.0 diff --git a/setup.py b/setup.py index 30ef9495..c8f22102 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ install_requires=[ 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', - 'blist>=1.3.6', + 'sortedcontainers>=2.2.2', 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', From ff65ee004ea17bb7963261e9eb79ec2f797f9401 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 10 Dec 2020 00:26:02 +0900 Subject: [PATCH 043/128] Kibana 7.10 --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index e2ecfb67..fc4cf0b8 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -553,9 +553,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10` -``kibana_discover_version: '7.8'`` +``kibana_discover_version: '7.10'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index b332facc..f846ac6c 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -14,7 +14,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index ba265a73..b8bcf8e4 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -217,7 +217,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index ae5c8bca..a508c25d 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.9', + 'kibana_discover_version': '7.10', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.9', + 'kibana_discover_version': '7.10', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.9', + 'kibana_discover_version': '7.10', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.9', + 'kibana_discover_version': '7.10', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.9', + 'kibana_discover_version': '7.10', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From b7dc75d1160d50c5e0c64e47f80c787279f25121 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 29 Dec 2020 15:12:09 +0900 Subject: [PATCH 044/128] Fix docker test --- Dockerfile-test | 2 ++ requirements-dev.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile-test b/Dockerfile-test index 3c153e64..fb8a7840 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,6 +1,8 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y +RUN apt-get install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert diff --git a/requirements-dev.txt b/requirements-dev.txt index 558761d9..d15887c0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ pylint<1.4 pytest<3.3.0 setuptools sphinx_rtd_theme -tox<2.0 +tox==3.20.1 From 474c69cd4084cd8a304afda9fa6a0af8d4adfffc Mon Sep 17 00:00:00 2001 From: Naoyuki Sano Date: Mon, 4 Jan 2021 21:23:43 +0900 Subject: [PATCH 045/128] Update loaders.py --- elastalert/loaders.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 08f06f64..f0751946 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,8 +77,10 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, - 'hivealerter': alerts.HiveAlerter, - 'linenotify': alerts.LineNotifyAlerter + 'pagertree': alerts.PagerTreeAlerter, + 'linenotify': alerts.LineNotifyAlerter, + 'hivealerter': alerts.HiveAlerter + } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list From d38ebcd0184e2622532a6b8e92108ed5e1ec0fb5 Mon Sep 17 00:00:00 2001 From: Naoyuki Sano Date: Tue, 12 Jan 2021 08:10:47 +0900 Subject: [PATCH 046/128] Update loaders.py --- elastalert/loaders.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index f0751946..01c178e3 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -79,8 +79,7 @@ class RulesLoader(object): 'post': alerts.HTTPPostAlerter, 'pagertree': alerts.PagerTreeAlerter, 'linenotify': alerts.LineNotifyAlerter, - 'hivealerter': alerts.HiveAlerter - + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list From f77b746f3416b92d189675325714954e3413ca9a Mon Sep 17 00:00:00 2001 From: Naoyuki Sano Date: Tue, 12 Jan 2021 08:26:20 +0900 Subject: [PATCH 047/128] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 91acfd3f..f2c11da5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,10 +5,10 @@ boto3>=1.4.4 cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 -elasticsearch>=7.0.0 +elasticsearch==7.0.0 envparse>=0.2.0 exotel>=0.1.3 -jira>=1.0.10,<1.0.15 +jira>=2.0.0 jsonschema>=3.0.2 mock>=2.0.0 prison>=0.1.2 From d22e3e531d49b46b4840e674ce9c45cee7d137d2 Mon Sep 17 00:00:00 2001 From: Naoyuki Sano Date: Tue, 12 Jan 2021 08:28:55 +0900 Subject: [PATCH 048/128] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 184d09bc..4bc0be73 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'jsonschema>=3.0.2', 'mock>=2.0.0', 'prison>=0.1.2', + 'py-zabbix>=1.1.3', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=5.1', @@ -47,7 +48,6 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', - 'cffi>=1.11.5', - 'py-zabbix>=1.1.3' + 'cffi>=1.11.5' ] ) From 84c93c88197910e6fa97eba8b1a22d6b0a6d5042 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Wed, 13 Jan 2021 01:19:29 +0900 Subject: [PATCH 049/128] Fix documents --- README.md | 13 ++++-- docs/source/elastalert.rst | 15 ++++++- docs/source/ruletypes.rst | 81 +++++++++++++++++++++++++++----------- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 99acc02e..cbaec79d 100644 --- a/README.md +++ b/README.md @@ -42,20 +42,27 @@ Currently, we have built-in support for the following alert types: - Email - JIRA - OpsGenie -- Commands +- Command - HipChat - MS Teams - Slack +- Mattermost - Telegram - GoogleChat - AWS SNS -- VictorOps +- Splunk On-Call (Formerly VictorOps) - PagerDuty - PagerTree - Exotel - Twilio - Gitter +- ServiceNow +- Debug +- Stomp +- Alerta +- HTTP POST - Line Notify +- TheHive - Zabbix Additional rule types and alerts can be easily imported or written. @@ -115,7 +122,7 @@ A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert in ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 \ +docker run -d -p 3030:3030 -p 3333:3333 \ -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ -v `pwd`/rules:/opt/elastalert/rules \ diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index b1008c3c..95e049b8 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -35,14 +35,27 @@ Currently, we have support built in for these alert types: - Email - JIRA - OpsGenie -- SNS +- AWS SNS +- MS Teams - HipChat - Slack +- Mattermost - Telegram - GoogleChat +- Splunk On-Call (Formerly VictorOps) +- PagerDuty +- PagerTree +- Exotel +- Twilio +- Gitter +- ServiceNow - Debug - Stomp +- Alerta +- HTTP POST +- Line Notify - TheHive +- Zabbix Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ff376371..200df5f8 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1620,7 +1620,7 @@ OpsGenie alerter will create an alert which can be used to notify Operations peo integration must be created in order to acquire the necessary ``opsgenie_key`` rule variable. Currently the OpsGenieAlerter only creates an alert, however it could be extended to update or close existing alerts. -It is necessary for the user to create an OpsGenie Rest HTTPS API `integration page `_ in order to create alerts. +It is necessary for the user to create an OpsGenie Rest HTTPS API `integration page `_ in order to create alerts. The OpsGenie alert requires one option: @@ -1632,7 +1632,7 @@ Optional: ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. ``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. -``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. +``opsgenie_default_receipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. ``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation). ``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data) ``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful. @@ -1650,6 +1650,8 @@ Optional: ``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. +``opsgenie_proxy``: By default ElastAlert will not use a network proxy to send notifications to OpsGenie. Set this option using ``hostname:port`` if you need to use a proxy. + Example usage:: opsgenie_details: @@ -1657,11 +1659,11 @@ Example usage:: Environment: '$VAR' # environment variable Message: { field: message } # field in the first match -SNS -~~~ +AWS SNS +~~~~~~~ -The SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. -The SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or +The AWS SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. +The AWS SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or via environment variables. See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for details. SNS requires one option: @@ -1670,13 +1672,22 @@ SNS requires one option: Optional: -``aws_access_key``: An access key to connect to SNS with. +``aws_access_key_id``: An access key to connect to SNS with. -``aws_secret_key``: The secret key associated with the access key. +``aws_secret_access_key``: The secret key associated with the access key. ``aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 -``profile``: The AWS profile to use. If none specified, the default will be used. +``aws_profile``: The AWS profile to use. If none specified, the default will be used. + +Example usage:: + + alert: + - sns: + aws_region: 'us-east-1' # You must nest aws_region within your alert configuration so it is not used to sign AWS requests. + sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' + aws_access_key_id: 'XXXXXXXXXXXXXXXXXX'' + aws_secret_access_key: 'YYYYYYYYYYYYYYYYYYYY' HipChat ~~~~~~~ @@ -1776,7 +1787,7 @@ Optional: ElastAlert rule. Any Apple emoji can be used, see http://emojipedia.org/apple/ . If slack_icon_url_override parameter is provided, emoji is ignored. ``slack_icon_url_override``: By default ElastAlert will use the :ghost: emoji when posting to the channel. You can provide icon_url to use custom image. -Provide absolute address of the pciture, for example: http://some.address.com/image.jpg . +Provide absolute address of the pciture. ``slack_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good' or 'warning' colors. @@ -1784,6 +1795,8 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. +``slack_ignore_ssl_errors``: By default ElastAlert will verify SSL certificate. Set this option to False if you want to ignore SSL errors. + ``slack_title``: Sets a title for the message, this shows up as a blue text at the start of the message ``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. @@ -1796,6 +1809,8 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``slack_ca_certs``: path to a CA cert bundle to use to verify SSL connections. + Mattermost ~~~~~~~~~~ @@ -1816,7 +1831,7 @@ Optional: ``mattermost_channel_override``: Incoming webhooks have a default channel, but it can be overridden. A public channel can be specified "#other-channel", and a Direct Message with "@username". ``mattermost_icon_url_override``: By default ElastAlert will use the default webhook icon when posting to the channel. You can provide icon_url to use custom image. -Provide absolute address of the picture (for example: http://some.address.com/image.jpg) or Base64 data url. +Provide absolute address of the picture or Base64 data url. ``mattermost_msg_pretext``: You can set the message attachment pretext using this option. @@ -1832,7 +1847,7 @@ Telegram alerter will send a notification to a predefined Telegram username or c The alerter requires the following two options: -``telegram_bot_token``: The token is a string along the lines of ``110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw`` that will be required to authorize the bot and send requests to the Bot API. You can learn about obtaining tokens and generating new ones in this document https://core.telegram.org/bots#botfather +``telegram_bot_token``: The token is a string along the lines of ``110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw`` that will be required to authorize the bot and send requests to the Bot API. You can learn about obtaining tokens and generating new ones in this document https://core.telegram.org/bots#6-botfather ``telegram_room_id``: Unique identifier for the target chat or username of the target channel using telegram chat_id (in the format "-xxxxxxxx") @@ -1842,6 +1857,10 @@ Optional: ``telegram_proxy``: By default ElastAlert will not use a network proxy to send notifications to Telegram. Set this option using ``hostname:port`` if you need to use a proxy. +``telegram_proxy_login``: The Telegram proxy auth username. + +``telegram_proxy_pass``: The Telegram proxy auth password. + GoogleChat ~~~~~~~~~~ GoogleChat alerter will send a notification to a predefined GoogleChat channel. The body of the notification is formatted the same as with other alerters. @@ -1893,7 +1912,7 @@ V2 API Options (Optional): These options are specific to the PagerDuty V2 API -See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 +See https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ ``pagerduty_api_version``: Defaults to `v1`. Set to `v2` to enable the PagerDuty V2 Event API. @@ -1924,6 +1943,8 @@ The alerter requires the following options: ``pagertree_integration_url``: URL generated by PagerTree for the integration. +``pagertree_proxy``: By default ElastAlert will not use a network proxy to send notifications to PagerTree. Set this option using hostname:port if you need to use a proxy. + Exotel ~~~~~~ @@ -1935,7 +1956,7 @@ The alerter requires the following option: ``exotel_auth_token``: Auth token assosiated with your Exotel account. -If you don't know how to find your accound sid and auth token, refer - http://support.exotel.in/support/solutions/articles/3000023019-how-to-find-my-exotel-token-and-exotel-sid- +If you don't know how to find your accound sid and auth token, refer - https://support.exotel.com/support/solutions/articles/3000023019-how-to-find-my-exotel-token-and-exotel-sid ``exotel_to_number``: The phone number where you would like send the notification. @@ -1962,26 +1983,26 @@ The alerter requires the following option: ``twilio_from_number``: Your twilio phone number from which message will be sent. -VictorOps -~~~~~~~~~ +Splunk On-Call (Formerly VictorOps) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -VictorOps alerter will trigger an incident to a predefined VictorOps routing key. The body of the notification is formatted the same as with other alerters. +Splunk On-Call (Formerly VictorOps) alerter will trigger an incident to a predefined Splunk On-Call (Formerly VictorOps) routing key. The body of the notification is formatted the same as with other alerters. The alerter requires the following options: ``victorops_api_key``: API key generated under the 'REST Endpoint' in the Integrations settings. -``victorops_routing_key``: VictorOps routing key to route the alert to. +``victorops_routing_key``: Splunk On-Call (Formerly VictorOps) routing key to route the alert to. -``victorops_message_type``: VictorOps field to specify severity level. Must be one of the following: INFO, WARNING, ACKNOWLEDGEMENT, CRITICAL, RECOVERY +``victorops_message_type``: Splunk On-Call (Formerly VictorOps) field to specify severity level. Must be one of the following: INFO, WARNING, ACKNOWLEDGEMENT, CRITICAL, RECOVERY Optional: -``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents throughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert. +``victorops_entity_id``: The identity of the incident used by Splunk On-Call (Formerly VictorOps) to correlate incidents throughout the alert lifecycle. If not defined, Splunk On-Call (Formerly VictorOps) will assign a random string to each alert. ``victorops_entity_display_name``: Human-readable name of alerting entity to summarize incidents without affecting the life-cycle workflow. -``victorops_proxy``: By default ElastAlert will not use a network proxy to send notifications to VictorOps. Set this option using ``hostname:port`` if you need to use a proxy. +``victorops_proxy``: By default ElastAlert will not use a network proxy to send notifications to Splunk On-Call (Formerly VictorOps). Set this option using ``hostname:port`` if you need to use a proxy. Gitter ~~~~~~ @@ -2006,7 +2027,7 @@ The ServiceNow alerter will create a ne Incident in ServiceNow. The body of the The alerter requires the following options: -``servicenow_rest_url``: The ServiceNow RestApi url, this will look like https://instancename.service-now.com/api/now/v1/table/incident +``servicenow_rest_url``: The ServiceNow RestApi url, this will look like https://developer.servicenow.com/dev.do#!/reference/api/orlando/rest/c_TableAPI#r_TableAPI-POST ``username``: The ServiceNow Username to access the api. @@ -2043,12 +2064,20 @@ Stomp This alert type will use the STOMP protocol in order to push a message to a broker like ActiveMQ or RabbitMQ. The message body is a JSON string containing the alert details. The default values will work with a pristine ActiveMQ installation. -Optional: +The alerter requires the following options: ``stomp_hostname``: The STOMP host to use, defaults to localhost. + ``stomp_hostport``: The STOMP port to use, defaults to 61613. + ``stomp_login``: The STOMP login to use, defaults to admin. + ``stomp_password``: The STOMP password to use, defaults to admin. + +Optional: + +``stomp_ssl``: Connect the STOMP host using TLS, defaults to False. + ``stomp_destination``: The STOMP destination to use, defaults to /queue/ALERT The stomp_destination field depends on the broker, the /queue/ALERT example is the nomenclature used by ActiveMQ. Each broker has its own logic. @@ -2057,7 +2086,7 @@ Alerta ~~~~~~ Alerta alerter will post an alert in the Alerta server instance through the alert API endpoint. -See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta JSON format. +See https://docs.alerta.io/en/latest/api/alert.html for more details on the Alerta JSON format. For Alerta 5.0 @@ -2073,6 +2102,8 @@ Optional: ``alerta_use_match_timestamp``: If true, it will use the timestamp of the first match as the ``createTime`` of the alert. otherwise, the current server time is used. +``alerta_api_skip_ssl``: Defaults to False. + ``alert_missing_value``: Text to replace any match field not found when formating strings. Defaults to ````. The following options dictate the values of the API JSON payload: @@ -2201,6 +2232,8 @@ Optional: ``hive_proxies``: Proxy configuration. +``hive_verify``: Wether or not to enable SSL certificate validation. Defaults to False. + ``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. Example usage:: From 9eaa68ef92d54718ab245b4b6525dac266c213d0 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Wed, 13 Jan 2021 21:43:57 +0900 Subject: [PATCH 050/128] Undo readme.md and elastalert.rst --- README.md | 13 +++---------- docs/source/elastalert.rst | 15 +-------------- docs/source/ruletypes.rst | 23 +++++++---------------- 3 files changed, 11 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index cbaec79d..99acc02e 100644 --- a/README.md +++ b/README.md @@ -42,27 +42,20 @@ Currently, we have built-in support for the following alert types: - Email - JIRA - OpsGenie -- Command +- Commands - HipChat - MS Teams - Slack -- Mattermost - Telegram - GoogleChat - AWS SNS -- Splunk On-Call (Formerly VictorOps) +- VictorOps - PagerDuty - PagerTree - Exotel - Twilio - Gitter -- ServiceNow -- Debug -- Stomp -- Alerta -- HTTP POST - Line Notify -- TheHive - Zabbix Additional rule types and alerts can be easily imported or written. @@ -122,7 +115,7 @@ A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert in ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 -p 3333:3333 \ +docker run -d -p 3030:3030 \ -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ -v `pwd`/rules:/opt/elastalert/rules \ diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 95e049b8..b1008c3c 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -35,27 +35,14 @@ Currently, we have support built in for these alert types: - Email - JIRA - OpsGenie -- AWS SNS -- MS Teams +- SNS - HipChat - Slack -- Mattermost - Telegram - GoogleChat -- Splunk On-Call (Formerly VictorOps) -- PagerDuty -- PagerTree -- Exotel -- Twilio -- Gitter -- ServiceNow - Debug - Stomp -- Alerta -- HTTP POST -- Line Notify - TheHive -- Zabbix Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 200df5f8..7d37f226 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1659,11 +1659,11 @@ Example usage:: Environment: '$VAR' # environment variable Message: { field: message } # field in the first match -AWS SNS -~~~~~~~ +SNS +~~~ -The AWS SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. -The AWS SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or +The SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. +The SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or via environment variables. See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for details. SNS requires one option: @@ -1672,22 +1672,13 @@ SNS requires one option: Optional: -``aws_access_key_id``: An access key to connect to SNS with. +``aws_access_key``: An access key to connect to SNS with. -``aws_secret_access_key``: The secret key associated with the access key. +``aws_secret_key``: The secret key associated with the access key. ``aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 -``aws_profile``: The AWS profile to use. If none specified, the default will be used. - -Example usage:: - - alert: - - sns: - aws_region: 'us-east-1' # You must nest aws_region within your alert configuration so it is not used to sign AWS requests. - sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' - aws_access_key_id: 'XXXXXXXXXXXXXXXXXX'' - aws_secret_access_key: 'YYYYYYYYYYYYYYYYYYYY' +``profile``: The AWS profile to use. If none specified, the default will be used. HipChat ~~~~~~~ From 29ea671cd05e6ad6807a26a52bc24af6e2ddd261 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 00:22:40 +0900 Subject: [PATCH 051/128] Remove hipchat & Fix loaders.py --- README.md | 1 - docs/source/elastalert.rst | 1 - docs/source/ruletypes.rst | 37 ---------------- elastalert/alerts.py | 86 -------------------------------------- elastalert/loaders.py | 10 +---- elastalert/schema.yaml | 9 ---- tests/alerts_test.py | 58 ------------------------- 7 files changed, 1 insertion(+), 201 deletions(-) diff --git a/README.md b/README.md index 99acc02e..6d91859e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Currently, we have built-in support for the following alert types: - JIRA - OpsGenie - Commands -- HipChat - MS Teams - Slack - Telegram diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 3bd400d0..2d962f1d 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -36,7 +36,6 @@ Currently, we have support built in for these alert types: - JIRA - OpsGenie - AWS SNS -- HipChat - Slack - Telegram - GoogleChat diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 27e1eefb..7341e7a8 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1709,43 +1709,6 @@ Example When to use aws_profile usage:: sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' sns_aws_profile: 'default' -HipChat -~~~~~~~ - -HipChat alerter will send a notification to a predefined HipChat room. The body of the notification is formatted the same as with other alerters. - -The alerter requires the following two options: - -``hipchat_auth_token``: The randomly generated notification token created by HipChat. Go to https://XXXXX.hipchat.com/account/api and use -'Create new token' section, choosing 'Send notification' in Scopes list. - -``hipchat_room_id``: The id associated with the HipChat room you want to send the alert to. Go to https://XXXXX.hipchat.com/rooms and choose -the room you want to post to. The room ID will be the numeric part of the URL. - -``hipchat_msg_color``: The color of the message background that is sent to HipChat. May be set to green, yellow or red. Default is red. - -``hipchat_domain``: The custom domain in case you have HipChat own server deployment. Default is api.hipchat.com. - -``hipchat_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false. - -``hipchat_proxy``: By default ElastAlert will not use a network proxy to send notifications to HipChat. Set this option using ``hostname:port`` if you need to use a proxy. - -``hipchat_notify``: When set to true, triggers a hipchat bell as if it were a user. Default is true. - -``hipchat_from``: When humans report to hipchat, a timestamp appears next to their name. For bots, the name is the name of the token. The from, instead of a timestamp, defaults to empty unless set, which you can do here. This is optional. - -``hipchat_message_format``: Determines how the message is treated by HipChat and rendered inside HipChat applications -html - Message is rendered as HTML and receives no special treatment. Must be valid HTML and entities must be escaped (e.g.: '&' instead of '&'). May contain basic tags: a, b, i, strong, em, br, img, pre, code, lists, tables. -text - Message is treated just like a message sent by a user. Can include @mentions, emoticons, pastes, and auto-detected URLs (Twitter, YouTube, images, etc). -Valid values: html, text. -Defaults to 'html'. - -``hipchat_mentions``: When using a ``html`` message format, it's not possible to mentions specific users using the ``@user`` syntax. -In that case, you can set ``hipchat_mentions`` to a list of users which will be first mentioned using a single text message, then the normal ElastAlert message will be sent to Hipchat. -If set, it will mention the users, no matter if the original message format is set to HTML or text. -Valid values: list of strings. -Defaults to ``[]``. - MS Teams ~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 9ef85294..29441701 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -966,92 +966,6 @@ def alert(self, matches): elastalert_logger.info("Sent sns notification to %s" % (self.sns_topic_arn)) -class HipChatAlerter(Alerter): - """ Creates a HipChat room notification for each alert """ - required_options = frozenset(['hipchat_auth_token', 'hipchat_room_id']) - - def __init__(self, rule): - super(HipChatAlerter, self).__init__(rule) - self.hipchat_msg_color = self.rule.get('hipchat_msg_color', 'red') - self.hipchat_message_format = self.rule.get('hipchat_message_format', 'html') - self.hipchat_auth_token = self.rule['hipchat_auth_token'] - self.hipchat_room_id = self.rule['hipchat_room_id'] - self.hipchat_domain = self.rule.get('hipchat_domain', 'api.hipchat.com') - self.hipchat_ignore_ssl_errors = self.rule.get('hipchat_ignore_ssl_errors', False) - self.hipchat_notify = self.rule.get('hipchat_notify', True) - self.hipchat_from = self.rule.get('hipchat_from', '') - self.url = 'https://%s/v2/room/%s/notification?auth_token=%s' % ( - self.hipchat_domain, self.hipchat_room_id, self.hipchat_auth_token) - self.hipchat_proxy = self.rule.get('hipchat_proxy', None) - - def create_alert_body(self, matches): - body = super(HipChatAlerter, self).create_alert_body(matches) - - # HipChat sends 400 bad request on messages longer than 10000 characters - if self.hipchat_message_format == 'html': - # Use appropriate line ending for text/html - br = '
' - body = body.replace('\n', br) - - truncated_message = '
...(truncated)' - truncate_to = 10000 - len(truncated_message) - else: - truncated_message = '..(truncated)' - truncate_to = 10000 - len(truncated_message) - - if (len(body) > 9999): - body = body[:truncate_to] + truncated_message - - return body - - def alert(self, matches): - body = self.create_alert_body(matches) - - # Post to HipChat - headers = {'content-type': 'application/json'} - # set https proxy, if it was provided - proxies = {'https': self.hipchat_proxy} if self.hipchat_proxy else None - payload = { - 'color': self.hipchat_msg_color, - 'message': body, - 'message_format': self.hipchat_message_format, - 'notify': self.hipchat_notify, - 'from': self.hipchat_from - } - - try: - if self.hipchat_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() - - if self.rule.get('hipchat_mentions', []): - ping_users = self.rule.get('hipchat_mentions', []) - ping_msg = payload.copy() - ping_msg['message'] = "ping {}".format( - ", ".join("@{}".format(user) for user in ping_users) - ) - ping_msg['message_format'] = "text" - - response = requests.post( - self.url, - data=json.dumps(ping_msg, cls=DateTimeEncoder), - headers=headers, - verify=not self.hipchat_ignore_ssl_errors, - proxies=proxies) - - response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, - verify=not self.hipchat_ignore_ssl_errors, - proxies=proxies) - warnings.resetwarnings() - response.raise_for_status() - except RequestException as e: - raise EAException("Error posting to HipChat: %s" % e) - elastalert_logger.info("Alert sent to HipChat room %s" % self.hipchat_room_id) - - def get_info(self): - return {'type': 'hipchat', - 'hipchat_room_id': self.hipchat_room_id} - - class MsTeamsAlerter(Alerter): """ Creates a Microsoft Teams Conversation Message for each alert """ required_options = frozenset(['ms_teams_webhook_url', 'ms_teams_alert_summary']) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 4ed8ccc7..0c3b13d7 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -62,7 +62,6 @@ class RulesLoader(object): 'debug': alerts.DebugAlerter, 'command': alerts.CommandAlerter, 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, 'mattermost': alerts.MattermostAlerter, @@ -78,7 +77,7 @@ class RulesLoader(object): 'post': alerts.HTTPPostAlerter, 'pagertree': alerts.PagerTreeAlerter, 'linenotify': alerts.LineNotifyAlerter, - 'hivealerter': alerts.HiveAlerter + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list @@ -316,13 +315,6 @@ def _dt_to_ts_with_format(dt): rule.setdefault('client_cert', conf.get('client_cert')) rule.setdefault('client_key', conf.get('client_key')) - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - # Make sure we have required options if self.required_locals - frozenset(list(rule.keys())): raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index eaff142f..eff1bbac 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -261,15 +261,6 @@ properties: jira_max_age: {type: number} jira_watchers: *arrayOfString - ### HipChat - hipchat_auth_token: {type: string} - hipchat_room_id: {type: [string, integer]} - hipchat_domain: {type: string} - hipchat_ignore_ssl_errors: {type: boolean} - hipchat_notify: {type: boolean} - hipchat_from: {type: string} - hipchat_mentions: {type: array, items: {type: string}} - ### Slack slack_webhook_url: *arrayOfString slack_username_override: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 59ee5746..e823d096 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -13,7 +13,6 @@ from elastalert.alerts import BasicMatchString from elastalert.alerts import CommandAlerter from elastalert.alerts import EmailAlerter -from elastalert.alerts import HipChatAlerter from elastalert.alerts import HTTPPostAlerter from elastalert.alerts import JiraAlerter from elastalert.alerts import JiraFormattedMatchString @@ -2085,63 +2084,6 @@ def test_resolving_rule_references(ea): assert 'the_owner' == alert.rule['nested_dict']['nested_owner'] -def test_hipchat_body_size_limit_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'hipchat_auth_token': 'token', - 'hipchat_room_id': 'room_id', - 'hipchat_message_format': 'text', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something.\n\n{message}', - 'alert_text_type': 'alert_text_only', - 'alert': [], - 'alert_text_kw': { - '@timestamp': 'time', - 'message': 'message', - }, - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HipChatAlerter(rule) - match = { - '@timestamp': '2018-01-01T00:00:00', - 'message': 'foo bar\n' * 5000, - } - body = alert.create_alert_body([match]) - - assert len(body) <= 10000 - - -def test_hipchat_body_size_limit_html(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'hipchat_auth_token': 'token', - 'hipchat_room_id': 'room_id', - 'hipchat_message_format': 'html', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something.\n\n{message}', - 'alert_text_type': 'alert_text_only', - 'alert': [], - 'alert_text_kw': { - '@timestamp': 'time', - 'message': 'message', - }, - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HipChatAlerter(rule) - match = { - '@timestamp': '2018-01-01T00:00:00', - 'message': 'foo bar\n' * 5000, - } - - body = alert.create_alert_body([match]) - - assert len(body) <= 10000 - - def test_alerta_no_auth(ea): rule = { 'name': 'Test Alerta rule!', From b30a58ee10f7a3ca07dd23d3869618f32478b2a6 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 02:48:30 +0900 Subject: [PATCH 052/128] Typo in example_rules/ssh.yaml --- example_rules/ssh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml index 7af89078..a7147217 100644 --- a/example_rules/ssh.yaml +++ b/example_rules/ssh.yaml @@ -1,5 +1,5 @@ # Rule name, must be unique - name: SSH abuse (ElastAlert 3.0.1) - 2 +name: SSH abuse (ElastAlert 3.0.1) - 2 # Alert on x events in y seconds type: frequency From a9a417c54d9a34de62ff8df5c2f217d5a26489b2 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 02:50:20 +0900 Subject: [PATCH 053/128] Remove duplicate property in example config file --- config.yaml.example | 1 - 1 file changed, 1 deletion(-) diff --git a/config.yaml.example b/config.yaml.example index 9d917638..c4921120 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -48,7 +48,6 @@ es_port: 9200 # Use SSL authentication with client certificates client_cert must be # a pem file containing both cert and key for client -#verify_certs: True #ca_certs: /path/to/cacert.pem #client_cert: /path/to/client_cert.pem #client_key: /path/to/client_key.key From 4b928d763825900f7a1e50d1b8c72ddd9d69d591 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 02:53:35 +0900 Subject: [PATCH 054/128] Fixed the logging property in config.yaml.example --- config.yaml.example | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index c4921120..89db954b 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -77,38 +77,38 @@ alert_time_limit: # logline: # format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s' # -# handlers: -# console: -# class: logging.StreamHandler -# formatter: logline -# level: DEBUG -# stream: ext://sys.stderr +# handlers: +# console: +# class: logging.StreamHandler +# formatter: logline +# level: DEBUG +# stream: ext://sys.stderr # -# file: -# class : logging.FileHandler -# formatter: logline -# level: DEBUG -# filename: elastalert.log +# file: +# class : logging.FileHandler +# formatter: logline +# level: DEBUG +# filename: elastalert.log # -# loggers: -# elastalert: -# level: WARN -# handlers: [] -# propagate: true +# loggers: +# elastalert: +# level: WARN +# handlers: [] +# propagate: true # -# elasticsearch: -# level: WARN -# handlers: [] -# propagate: true +# elasticsearch: +# level: WARN +# handlers: [] +# propagate: true # -# elasticsearch.trace: -# level: WARN -# handlers: [] -# propagate: true +# elasticsearch.trace: +# level: WARN +# handlers: [] +# propagate: true # -# '': # root logger -# level: WARN -# handlers: -# - console -# - file -# propagate: false +# '': # root logger +# level: WARN +# handlers: +# - console +# - file +# propagate: false From 5a5c49a40f1a59f9755a9b39b845b75963204917 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 23:23:33 +0900 Subject: [PATCH 055/128] add tzlocal<3.0 --- requirements.txt | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e70415b2..4f23de5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ requests>=2.10.0 stomp.py>=4.1.17 texttable>=0.8.8 twilio>=6.0.0,<6.1 +tzlocal<3.0 diff --git a/setup.py b/setup.py index b7ca722f..4591f27d 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', - 'cffi>=1.11.5' + 'cffi>=1.11.5', + 'tzlocal<3.0' ] ) From 3bc7297a1c8add1fa9f2a8b3ea12bfdd4cafa0e7 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 14 Jan 2021 23:49:03 +0900 Subject: [PATCH 056/128] Update running_elastalert.rst --- docs/source/running_elastalert.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 7fdf1eeb..91b8f862 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -10,7 +10,10 @@ Requirements - ISO8601 or Unix timestamped data - Python 3.6 - pip, see requirements.txt -- Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev +- Packages on Ubuntu 18.x: build-essential python3-pip python3.6 python3.6-dev libffi-dev libssl-dev +- Packages on Ubuntu 20.x: build-essential python3-pip python3.6 python3.6-dev libffi-dev libssl-dev +- Packages on CentOS 7.x: 'Development Tools' python3-pip python36 python3-devel python3-libs python3-setuptools libffi-devel openssl-devel +- Packages on CentOS 8.x: 'Development Tools' python3-pip python36 python3-devel python3-setuptools python3-libs libffi-devel openssl-devel Downloading and Configuring --------------------------- From 26cc8a6e9c439d12a50aaa4499af6bb804996744 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 00:00:04 +0900 Subject: [PATCH 057/128] add opsgenie_addr to docs --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 7341e7a8..d818af8e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1629,7 +1629,7 @@ The OpsGenie alert requires one option: Optional: ``opsgenie_account``: The OpsGenie account to integrate with. - +``opsgenie_addr``: The OpsGenie URL to to connect against, default is ``https://api.opsgenie.com/v2/alerts`` ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. ``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. ``opsgenie_default_receipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. From bb4631430e33874b3b974f80295f6da1fa098186 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 00:09:51 +0900 Subject: [PATCH 058/128] fix ruletypes.rst typo --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 7341e7a8..ab8ecbd7 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -938,7 +938,7 @@ Optional: ``field_value``: When set, uses the value of the field in the document and not the number of matching documents. This is useful to monitor for example a temperature sensor and raise an alarm if the temperature grows too fast. Note that the means of the field on the reference and current windows are used to determine if the ``spike_height`` value is reached. -Note also that the threshold parameters are ignored in this smode. +Note also that the threshold parameters are ignored in this mode. ``threshold_ref``: The minimum number of events that must exist in the reference window for an alert to trigger. For example, if From 9edf294c8f2f6d4e9614e2c557feb5d452572f35 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 00:18:31 +0900 Subject: [PATCH 059/128] Remove Duplicate Key in Schema YAML --- elastalert/schema.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index eff1bbac..e9aca22d 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -347,7 +347,6 @@ properties: alerta_origin: {type: string} # Python format string alerta_group: {type: string} # Python format string alerta_service: {type: array, items: {type: string}} # Python format string - alerta_service: {type: array, items: {type: string}} # Python format string alerta_correlate: {type: array, items: {type: string}} # Python format string alerta_tags: {type: array, items: {type: string}} # Python format string alerta_event: {type: string} # Python format string From 62efda09cc2794c3f9e54aed2d323ccd7891f69f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 00:41:05 +0900 Subject: [PATCH 060/128] fix zabbix --- elastalert/loaders.py | 4 ++- elastalert/zabbix.py | 60 ++++++++++++++++++++++++++++--------------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 0c3b13d7..a0361075 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -25,6 +25,7 @@ from .util import ts_to_dt_with_format from .util import unix_to_dt from .util import unixms_to_dt +from .zabbix import ZabbixAlerter class RulesLoader(object): @@ -77,7 +78,8 @@ class RulesLoader(object): 'post': alerts.HTTPPostAlerter, 'pagertree': alerts.PagerTreeAlerter, 'linenotify': alerts.LineNotifyAlerter, - 'hivealerter': alerts.HiveAlerter + 'hivealerter': alerts.HiveAlerter, + 'zabbix': ZabbixAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index e3f13aa0..e2b5f1ed 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -1,46 +1,47 @@ -from alerts import Alerter # , BasicMatchString -import logging -from pyzabbix.api import ZabbixAPI -from pyzabbix import ZabbixSender, ZabbixMetric from datetime import datetime +from pyzabbix import ZabbixSender, ZabbixMetric, ZabbixAPI + +from .alerts import Alerter +from .util import elastalert_logger, EAException + class ZabbixClient(ZabbixAPI): - def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost', - sender_port=10051): + def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', + sender_host='localhost', sender_port=10051): self.url = url self.use_authenticate = use_authenticate self.sender_host = sender_host self.sender_port = sender_port self.metrics_chunk_size = 200 self.aggregated_metrics = [] - self.logger = logging.getLogger(self.__class__.__name__) - super(ZabbixClient, self).__init__(url=self.url, use_authenticate=self.use_authenticate, user=user, password=password) + + super(ZabbixClient, self).__init__(url=self.url, + use_authenticate=self.use_authenticate, + user=user, + password=password) def send_metric(self, hostname, key, data): zm = ZabbixMetric(hostname, key, data) if self.send_aggregated_metrics: - self.aggregated_metrics.append(zm) if len(self.aggregated_metrics) > self.metrics_chunk_size: - self.logger.info("Sending: %s metrics" % (len(self.aggregated_metrics))) + elastalert_logger.info("Sending: %s metrics" % (len(self.aggregated_metrics))) try: - ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(self.aggregated_metrics) + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port) \ + .send(self.aggregated_metrics) self.aggregated_metrics = [] except Exception as e: - self.logger.exception(e) - pass + elastalert_logger.exception(e) else: try: - ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(zm) + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send([zm]) except Exception as e: - self.logger.exception(e) - pass + elastalert_logger.exception(e) class ZabbixAlerter(Alerter): - # By setting required_options to a set of strings # You can ensure that the rule config file specifies all # of the options. Otherwise, ElastAlert will throw an exception @@ -54,6 +55,9 @@ def __init__(self, *args): self.zbx_sender_port = self.rule.get('zbx_sender_port', 10051) self.zbx_host = self.rule.get('zbx_host') self.zbx_key = self.rule.get('zbx_key') + self.timestamp_field = self.rule.get('timestamp_field', '@timestamp') + self.timestamp_type = self.rule.get('timestamp_type', 'iso') + self.timestamp_strptime = self.rule.get('timestamp_strptime', '%Y-%m-%dT%H:%M:%S.%fZ') # Alert is called def alert(self, matches): @@ -63,10 +67,26 @@ def alert(self, matches): # the aggregation option set zm = [] for match in matches: - ts_epoch = int(datetime.strptime(match['@timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) - zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) + if ':' not in match[self.timestamp_field] or '-' not in match[self.timestamp_field]: + ts_epoch = int(match[self.timestamp_field]) + else: + try: + ts_epoch = int(datetime.strptime(match[self.timestamp_field], self.timestamp_strptime) + .strftime('%s')) + except ValueError: + ts_epoch = int(datetime.strptime(match[self.timestamp_field], '%Y-%m-%dT%H:%M:%SZ') + .strftime('%s')) + zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value='1', clock=ts_epoch)) - ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + try: + response = ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + if response.failed: + elastalert_logger.warning("Missing zabbix host '%s' or host's item '%s', alert will be discarded" + % (self.zbx_host, self.zbx_key)) + else: + elastalert_logger.info("Alert sent to Zabbix") + except Exception as e: + raise EAException("Error sending alert to Zabbix: %s" % e) # get_info is called after an alert is sent to get data that is written back # to Elasticsearch in the field "alert_info" From d45a83f54007d4a6a75459fe5e5ef15f4819573b Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 01:22:44 +0900 Subject: [PATCH 061/128] update alert types and Third Party Tools --- README.md | 33 +++++++++++++++++++++++++++++---- docs/source/elastalert.rst | 13 +++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 604cc081..fc2461bf 100644 --- a/README.md +++ b/README.md @@ -39,22 +39,29 @@ Several rule types with common monitoring paradigms are included with ElastAlert Currently, we have built-in support for the following alert types: +- Command - Email - JIRA - OpsGenie -- Commands +- AWS SNS - MS Teams - Slack +- Mattermost - Telegram - GoogleChat -- AWS SNS -- VictorOps - PagerDuty - PagerTree - Exotel - Twilio +- Splunk On-Call (Formerly VictorOps) - Gitter +- ServiceNow +- Debug +- Stomp +- Alerta +- HTTP POST - Line Notify +- TheHive - Zabbix Additional rule types and alerts can be easily imported or written. @@ -114,7 +121,7 @@ A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert in ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 \ +docker run -d -p 3030:3030 -p 3333:3333 \ -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ -v `pwd`/rules:/opt/elastalert/rules \ @@ -123,6 +130,24 @@ docker run -d -p 3030:3030 \ --name elastalert bitsensor/elastalert:latest ``` +### ElastAlert-Docker + +This Dockerfile will build a Docker image for Elastalert. This image is compatible with the accompanying Helm chart for Kubernetes. + +https://github.com/jertel/elastalert-docker + +### ElastAlert uses ElastAlert Helm Chart almost as it is *Use original Docker Image to fix bugs + +[ElastAlert Server Helm Chart](https://github.com/daichi703n/elastalert-helm) + +[Praeco Helm Chart](https://github.com/daichi703n/praeco-helm) + +[Installing Praeco (ElastAlert GUI) into Kubernetes with Helm](https://en-designetwork.daichi703n.com/entry/2020/02/24/praeco-helm-kubernetes) + +### Praeco +![Praeco screenshot](https://user-images.githubusercontent.com/611996/47752071-7c4a9080-dc61-11e8-8ccf-2196f13429b2.png) +[Praeco](https://github.com/johnsusek/praeco) is a free open source GUI for ElastAlert. + ## Documentation Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index d6654b85..71c1b8cb 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -36,12 +36,25 @@ Currently, we have support built in for these alert types: - JIRA - OpsGenie - AWS SNS +- MS Teams - Slack +- Mattermost - Telegram - GoogleChat +- PagerDuty +- PagerTree +- Exotel +- Twilio +- Splunk On-Call (Formerly VictorOps) +- Gitter +- ServiceNow - Debug - Stomp +- Alerta +- HTTP POST +- Line Notify - TheHive +- Zabbix Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) From 84888b03a1e34bae069cbe1af5cac6a26bd82ef4 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 02:12:01 +0900 Subject: [PATCH 062/128] Fix Logging inconsistencies --- elastalert/alerts.py | 15 +++++++-------- elastalert/elastalert.py | 32 ++++++++++++++++---------------- elastalert/kibana_discover.py | 9 +++++---- elastalert/loaders.py | 14 +++++++------- elastalert/opsgenie.py | 15 +++++++-------- elastalert/util.py | 6 +++--- 6 files changed, 45 insertions(+), 46 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index c4fef858..65b0b5de 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2,7 +2,6 @@ import copy import datetime import json -import logging import os import re import subprocess @@ -590,7 +589,7 @@ def __init__(self, rule): msg = '%s Both have common statuses of (%s). As such, no tickets will ever be found.' % ( msg, ','.join(intersection)) msg += ' This should be simplified to use only one or the other.' - logging.warning(msg) + elastalert_logger.warning(msg) self.reset_jira_args() @@ -610,7 +609,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) + elastalert_logger.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -753,7 +752,7 @@ def find_existing_ticket(self, matches): try: issues = self.client.search_issues(jql) except JIRAError as e: - logging.exception("Error while searching for JIRA ticket using jql '%s': %s" % (jql, e)) + elastalert_logger.exception("Error while searching for JIRA ticket using jql '%s': %s" % (jql, e)) return None if len(issues): @@ -796,19 +795,19 @@ def alert(self, matches): try: self.comment_on_ticket(ticket, match) except JIRAError as e: - logging.exception("Error while commenting on ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while commenting on ticket %s: %s" % (ticket, e)) if self.labels: for label in self.labels: try: ticket.fields.labels.append(label) except JIRAError as e: - logging.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) if self.transition: elastalert_logger.info('Transitioning existing ticket %s' % (ticket.key)) try: self.transition_ticket(ticket) except JIRAError as e: - logging.exception("Error while transitioning ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while transitioning ticket %s: %s" % (ticket, e)) if self.pipeline is not None: self.pipeline['jira_ticket'] = ticket @@ -899,7 +898,7 @@ def __init__(self, *args): if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: - logging.warning('Warning! You could be vulnerable to shell injection!') + elastalert_logger.warning('Warning! You could be vulnerable to shell injection!') self.rule['command'] = [self.rule['command']] self.new_style_string_format = False diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 81d2ecff..d9b1f87a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -407,7 +407,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): # Different versions of ES have this formatted in different ways. Fallback to str-ing the whole thing raise ElasticsearchException(str(res['_shards']['failures'])) - logging.debug(str(res)) + elastalert_logger.debug(str(res)) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -850,7 +850,7 @@ def enhance_filter(self, rule): filters.append(query_str_filter) else: filters.append({'query': query_str_filter}) - logging.debug("Enhanced filter with {} terms: {}".format(listname, str(query_str_filter))) + elastalert_logger.debug("Enhanced filter with {} terms: {}".format(listname, str(query_str_filter))) def run_rule(self, rule, endtime, starttime=None): """ Run a rule for a given time period, including querying and alerting on results. @@ -879,7 +879,7 @@ def run_rule(self, rule, endtime, starttime=None): # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: - logging.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) + elastalert_logger.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) return 0 # Run the rule. If querying over a large time period, split it up into segments @@ -1088,7 +1088,7 @@ def load_rule_changes(self): try: new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if not new_rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) @@ -1128,7 +1128,7 @@ def load_rule_changes(self): try: new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if not new_rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue @@ -1211,12 +1211,12 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): time.sleep(1.0) if self.writeback_es.ping(): - logging.error( + elastalert_logger.error( 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', self.writeback_alias, ) else: - logging.error( + elastalert_logger.error( 'Could not reach ElasticSearch at "%s:%d".', self.conf['es_host'], self.conf['es_port'], @@ -1291,7 +1291,7 @@ def handle_rule_execution(self, rule): # We were processing for longer than our refresh interval # This can happen if --start was specified with a large time period # or if we are running too slow to process events in real time. - logging.warning( + elastalert_logger.warning( "Querying from %s to %s took longer than %s!" % ( old_starttime, pretty_ts(endtime, rule.get('use_local_time')), @@ -1624,7 +1624,7 @@ def writeback(self, doc_type, body, rule=None, match_body=None): res = self.writeback_es.index(index=index, doc_type=doc_type, body=body) return res except ElasticsearchException as e: - logging.exception("Error writing alert info to Elasticsearch: %s" % (e)) + elastalert_logger.exception("Error writing alert info to Elasticsearch: %s" % (e)) def find_recent_pending_alerts(self, time_limit): """ Queries writeback_es to find alerts that did not send @@ -1652,7 +1652,7 @@ def find_recent_pending_alerts(self, time_limit): if res['hits']['hits']: return res['hits']['hits'] except ElasticsearchException as e: - logging.exception("Error finding recent pending alerts: %s %s" % (e, query)) + elastalert_logger.exception("Error finding recent pending alerts: %s %s" % (e, query)) return [] def send_pending_alerts(self): @@ -1852,11 +1852,11 @@ def add_aggregated_alert(self, match, rule): def silence(self, silence_cache_key=None): """ Silence an alert for a period of time. --silence and --rule must be passed as args. """ if self.debug: - logging.error('--silence not compatible with --debug') + elastalert_logger.error('--silence not compatible with --debug') exit(1) if not self.args.rule: - logging.error('--silence must be used with --rule') + elastalert_logger.error('--silence must be used with --rule') exit(1) # With --rule, self.rules will only contain that specific rule @@ -1869,11 +1869,11 @@ def silence(self, silence_cache_key=None): try: silence_ts = parse_deadline(self.args.silence) except (ValueError, TypeError): - logging.error('%s is not a valid time period' % (self.args.silence)) + elastalert_logger.error('%s is not a valid time period' % (self.args.silence)) exit(1) if not self.set_realert(silence_cache_key, silence_ts, 0): - logging.error('Failed to save silence command to Elasticsearch') + elastalert_logger.error('Failed to save silence command to Elasticsearch') exit(1) elastalert_logger.info('Success. %s will be silenced until %s' % (silence_cache_key, silence_ts)) @@ -1934,7 +1934,7 @@ def is_silenced(self, rule_name): def handle_error(self, message, data=None): ''' Logs message at error level and writes message, data and traceback to Elasticsearch. ''' - logging.error(message) + elastalert_logger.error(message) body = {'message': message} tb = traceback.format_exc() body['traceback'] = tb.strip().split('\n') @@ -1944,7 +1944,7 @@ def handle_error(self, message, data=None): def handle_uncaught_exception(self, exception, rule): """ Disables a rule and sends a notification. """ - logging.error(traceback.format_exc()) + elastalert_logger.error(traceback.format_exc()) self.handle_error('Uncaught exception running rule %s: %s' % (rule['name'], exception), {'rule': rule['name']}) if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index f846ac6c..ed8b681d 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -8,6 +8,7 @@ import urllib.parse from .util import EAException +from .util import elastalert_logger from .util import lookup_es_key from .util import ts_add @@ -21,7 +22,7 @@ def generate_kibana_discover_url(rule, match): discover_app_url = rule.get('kibana_discover_app_url') if not discover_app_url: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_app_url for rule %s' % ( rule.get('name', '') ) @@ -30,7 +31,7 @@ def generate_kibana_discover_url(rule, match): kibana_version = rule.get('kibana_discover_version') if not kibana_version: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_version for rule %s' % ( rule.get('name', '') ) @@ -39,7 +40,7 @@ def generate_kibana_discover_url(rule, match): index = rule.get('kibana_discover_index_pattern_id') if not index: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_index_pattern_id for rule %s' % ( rule.get('name', '') ) @@ -70,7 +71,7 @@ def generate_kibana_discover_url(rule, match): appState = kibana_discover_app_state(index, columns, filters, query_keys, match) else: - logging.warning( + elastalert_logger.warning( 'Unknown kibana discover application version %s for rule %s' % ( kibana_version, rule.get('name', '') diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 044adce9..c3487f9b 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -2,7 +2,6 @@ import copy import datetime import hashlib -import logging import os import sys @@ -21,6 +20,7 @@ from .util import dt_to_unix from .util import dt_to_unixms from .util import EAException +from .util import elastalert_logger from .util import get_module from .util import ts_to_dt from .util import ts_to_dt_with_format @@ -116,7 +116,7 @@ def load(self, conf, args=None): rule = self.load_configuration(rule_file, conf, args) # A rule failed to load, don't try to process it if not rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue # By setting "is_enabled: False" in rule file, a rule is easily disabled if 'is_enabled' in rule and not rule['is_enabled']: @@ -388,10 +388,10 @@ def _dt_to_ts_with_format(dt): if rule.get('use_strftime_index'): for token in ['%y', '%M', '%D']: if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime( - rule.get('index')))) + elastalert_logger.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') @@ -484,7 +484,7 @@ def adjust_deprecated_values(rule): rule['http_post_proxy'] = rule['simple_proxy'] if 'simple_webhook_url' in rule: rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning( + elastalert_logger.warning( '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index bcdaf2d0..8db52d89 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import json -import logging import os.path import requests @@ -46,11 +45,11 @@ def _parse_responders(self, responders, responder_args, matches, default_respond try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: - logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) + elastalert_logger.warning("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) if not formated_responders: - logging.warn("OpsGenieAlerter: no responders can be formed. Trying the default responder ") + elastalert_logger.warning("OpsGenieAlerter: no responders can be formed. Trying the default responder ") if not default_responders: - logging.warn("OpsGenieAlerter: default responder not set. Falling back") + elastalert_logger.warning("OpsGenieAlerter: default responder not set. Falling back") formated_responders = responders else: formated_responders = default_responders @@ -90,7 +89,7 @@ def alert(self, matches): post['tags'] = self.tags if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'): - logging.warn("Priority level does not appear to be specified correctly. \ + elastalert_logger.warning("Priority level does not appear to be specified correctly. \ Please make sure to set it to a value between P1 and P5") else: post['priority'] = self.priority @@ -102,7 +101,7 @@ def alert(self, matches): if details: post['details'] = details - logging.debug(json.dumps(post)) + elastalert_logger.debug(json.dumps(post)) headers = { 'Content-Type': 'application/json', @@ -114,12 +113,12 @@ def alert(self, matches): try: r = requests.post(self.to_addr, json=post, headers=headers, proxies=proxies) - logging.debug('request response: {0}'.format(r)) + elastalert_logger.debug('request response: {0}'.format(r)) if r.status_code != 202: elastalert_logger.info("Error response from {0} \n " "API Response: {1}".format(self.to_addr, r)) r.raise_for_status() - logging.info("Alert sent to OpsGenie") + elastalert_logger.info("Alert sent to OpsGenie") except Exception as err: raise EAException("Error sending alert: {0}".format(err)) diff --git a/elastalert/util.py b/elastalert/util.py index bbb0600f..3e9c9f66 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -152,7 +152,7 @@ def ts_to_dt(timestamp): def dt_to_ts(dt): if not isinstance(dt, datetime.datetime): - logging.warning('Expected datetime, got %s' % (type(dt))) + elastalert_logger.warning('Expected datetime, got %s' % (type(dt))) return dt ts = dt.isoformat() # Round microseconds to milliseconds @@ -176,7 +176,7 @@ def ts_to_dt_with_format(timestamp, ts_format): def dt_to_ts_with_format(dt, ts_format): if not isinstance(dt, datetime.datetime): - logging.warning('Expected datetime, got %s' % (type(dt))) + elastalert_logger.warning('Expected datetime, got %s' % (type(dt))) return dt ts = dt.strftime(ts_format) return ts @@ -361,7 +361,7 @@ def build_es_conn_config(conf): # Deprecated if 'boto_profile' in conf: - logging.warning('Found deprecated "boto_profile", use "profile" instead!') + elastalert_logger.warning('Found deprecated "boto_profile", use "profile" instead!') parsed_conf['profile'] = conf['boto_profile'] if 'profile' in conf: From 72201dae06b009693359b4843c74947e063cbf52 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 20:12:01 +0900 Subject: [PATCH 063/128] Fixing slack ssl verification --- elastalert/alerts.py | 2 +- tests/alerts_test.py | 63 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 65b0b5de..d2aac654 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1127,7 +1127,7 @@ def alert(self, matches): if self.slack_ca_certs: verify = self.slack_ca_certs else: - verify = self.slack_ignore_ssl_errors + verify = not self.slack_ignore_ssl_errors if self.slack_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() payload['channel'] = channel_override diff --git a/tests/alerts_test.py b/tests/alerts_test.py index e823d096..ba5a8c5d 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1240,7 +1240,7 @@ def test_slack_uses_custom_title(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1286,7 +1286,7 @@ def test_slack_uses_custom_timeout(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=20 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1330,7 +1330,7 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1375,7 +1375,7 @@ def test_slack_uses_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1436,7 +1436,7 @@ def test_slack_uses_list_of_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1487,7 +1487,7 @@ def test_slack_attach_kibana_discover_url_when_generated(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1532,7 +1532,7 @@ def test_slack_attach_kibana_discover_url_when_not_generated(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1584,7 +1584,7 @@ def test_slack_kibana_discover_title(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1636,13 +1636,58 @@ def test_slack_kibana_discover_color(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=False, + verify=True, timeout=10 ) actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data +def test_slack_ignore_ssl_errors(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'slack_ignore_ssl_errors': True, + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + + expected_data = { + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', From deca0b006a4b71cbb28c79d69fabb54cdd32c646 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 15 Jan 2021 20:20:21 +0900 Subject: [PATCH 064/128] Fix is_enabled not work with reload --- elastalert/elastalert.py | 17 ++++++++++++++--- elastalert/loaders.py | 3 --- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index d9b1f87a..db2e2749 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -176,7 +176,10 @@ def __init__(self, args): remove = [] for rule in self.rules: - if not self.init_rule(rule): + if 'is_enabled' in rule and not rule['is_enabled']: + self.disabled_rules.append(rule) + remove.append(rule) + elif not self.init_rule(rule): remove.append(rule) list(map(self.rules.remove, remove)) @@ -974,7 +977,7 @@ def run_rule(self, rule, endtime, starttime=None): def init_rule(self, new_rule, new=True): ''' Copies some necessary non-config state from an exiting rule to a new rule. ''' - if not new: + if not new and self.scheduler.get_job(job_id=new_rule['name']): self.scheduler.remove_job(job_id=new_rule['name']) try: @@ -1094,6 +1097,15 @@ def load_rule_changes(self): elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file] + # Stop job if is running + if self.scheduler.get_job(job_id=new_rule['name']): + self.scheduler.remove_job(job_id=new_rule['name']) + # Append to disabled_rule + for disabled_rule in self.disabled_rules: + if disabled_rule['name'] == new_rule['name']: + break + else: + self.disabled_rules.append(new_rule) continue except EAException as e: message = 'Could not load rule %s: %s' % (rule_file, e) @@ -1112,7 +1124,6 @@ def load_rule_changes(self): # Re-enable if rule had been disabled for disabled_rule in self.disabled_rules: if disabled_rule['name'] == new_rule['name']: - self.rules.append(disabled_rule) self.disabled_rules.remove(disabled_rule) break diff --git a/elastalert/loaders.py b/elastalert/loaders.py index dae799be..fc1bb483 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -120,9 +120,6 @@ def load(self, conf, args=None): if not rule: elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue if rule['name'] in names: raise EAException('Duplicate rule named %s' % (rule['name'])) except EAException as e: From c41e610c502379464122572cfcd40f9cb208ef64 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 18 Jan 2021 01:59:36 +0900 Subject: [PATCH 065/128] add discord alerter --- README.md | 1 + docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 28 ++++++++++++ elastalert/alerts.py | 62 ++++++++++++++++++++++++++ elastalert/loaders.py | 3 +- elastalert/schema.yaml | 3 ++ example_rules/exemple_discord_any.yaml | 40 +++++++++++++++++ 7 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 example_rules/exemple_discord_any.yaml diff --git a/README.md b/README.md index fc2461bf..7abb986e 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Currently, we have built-in support for the following alert types: - Line Notify - TheHive - Zabbix +- Discord Additional rule types and alerts can be easily imported or written. diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 71c1b8cb..f6683d55 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -55,6 +55,7 @@ Currently, we have support built in for these alert types: - Line Notify - TheHive - Zabbix +- Discord Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index e26d4fcf..d7a19803 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2282,3 +2282,31 @@ Required: ``zbx_sender_port``: The port where zabbix server is listenning. ``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert. ``zbx_key``: This field setup the key in the host that receives the value sent by Elastalert. + + +Discord +~~~~~~~ + +Discord will send notification to a Line application. The body of the notification is formatted the same as with other alerters. + +Required: + +``discord_webhook_url``: The webhook URL. + +Optional: + +``discord_emoji_title``: By default ElastAlert will use the ``:warning:`` emoji when posting to the channel. You can use a different emoji per ElastAlert rule. Any Apple emoji can be used, see http://emojipedia.org/apple/ . If slack_icon_url_override parameter is provided, emoji is ignored. + +``discord_http_proxy``: By default ElastAlert will not use a network http proxy to send notifications to Discord. Set this option using hostname:port if you need to use a proxy. + +``discord_https_proxy``: By default ElastAlert will not use a network https proxy to send notifications to Discord. Set this option using hostname:port if you need to use a proxy. + +``discord_proxy_login``: The Discord proxy auth username. + +``discord_proxy_password``: The Discord proxy auth username. + +``discord_embed_color``: embed color. By default ``0xffffff``. + +``discord_embed_footer``: embed footer. + +``discord_embed_icon_url``: You can provide icon_url to use custom image. Provide absolute address of the pciture. diff --git a/elastalert/alerts.py b/elastalert/alerts.py index a0801d8b..e44cdd68 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2010,3 +2010,65 @@ def get_info(self): 'type': 'hivealerter', 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') } + + +class DiscordAlerter(Alerter): + + required_options = frozenset(['discord_webhook_url']) + + def __init__(self, rule): + super(DiscordAlerter, self).__init__(rule) + self.discord_webhook_url = self.rule['discord_webhook_url'] + self.discord_emoji_title = self.rule.get('discord_emoji_title', ':warning:') + self.discord_proxy = self.rule.get('discord_proxy', None) + self.discord_proxy_login = self.rule.get('discord_proxy_login', None) + self.discord_proxy_password = self.rule.get('discord_proxy_password', None) + self.discord_embed_color = self.rule.get('discord_embed_color', 0xffffff) + self.discord_embed_footer = self.rule.get('discord_embed_footer', None) + self.discord_embed_icon_url = self.rule.get('discord_embed_icon_url', None) + + def alert(self, matches): + body = '' + title = u'%s' % (self.create_title(matches)) + for match in matches: + body += str(BasicMatchString(self.rule, match)) + if len(matches) > 1: + body += '\n----------------------------------------\n' + if len(body) > 2047: + body = body[0:1950] + '\n *message was cropped according to discord embed description limits!* ' + + body += '```' + + proxies = {'https': self.discord_proxy} if self.discord_proxy else None + auth = HTTPProxyAuth(self.discord_proxy_login, self.discord_proxy_password) if self.discord_proxy_login else None + headers = {"Content-Type": "application/json"} + + data = {} + data["content"] = "%s %s %s" % (self.discord_emoji_title, title, self.discord_emoji_title) + data["embeds"] = [] + embed = {} + embed["description"] = "%s" % (body) + embed["color"] = (self.discord_embed_color) + + if self.discord_embed_footer: + embed["footer"] = {} + embed["footer"]["text"] = (self.discord_embed_footer) if self.discord_embed_footer else None + embed["footer"]["icon_url"] = (self.discord_embed_icon_url) if self.discord_embed_icon_url else None + else: + None + + data["embeds"].append(embed) + + try: + response = requests.post(self.discord_webhook_url, data=json.dumps(data), headers=headers, proxies=proxies, auth=auth) + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Discord: %s. Details: %s" % (e, "" if e.response is None else e.response.text)) + + elastalert_logger.info( + "Alert sent to the webhook %s" % self.discord_webhook_url) + + def get_info(self): + return {'type': 'discord', + 'discord_webhook_url': self.discord_webhook_url} diff --git a/elastalert/loaders.py b/elastalert/loaders.py index fc1bb483..f2b1701b 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -80,7 +80,8 @@ class RulesLoader(object): 'pagertree': alerts.PagerTreeAlerter, 'linenotify': alerts.LineNotifyAlerter, 'hivealerter': alerts.HiveAlerter, - 'zabbix': ZabbixAlerter + 'zabbix': ZabbixAlerter, + 'discord': alerts.DiscordAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 474dd075..d249b327 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -371,3 +371,6 @@ properties: zbx_sender_port: {type: integer} zbx_host: {type: string} zbx_key: {type: string} + + ## Discord + discord_webhook_url: {type: string} diff --git a/example_rules/exemple_discord_any.yaml b/example_rules/exemple_discord_any.yaml new file mode 100644 index 00000000..e19b4a4b --- /dev/null +++ b/example_rules/exemple_discord_any.yaml @@ -0,0 +1,40 @@ +# This exemple will provide you every alerts that occured between the sleeping time your configured in your config file. +# Every match will be send as a unique alert to discord. If you got 3 match, the alerter will send 3 alert to your discord. + +name: "Exemple discord webhook alert" +type: any +index: your_indice_%Y-%m-%d +use_strftime_index: true + +# Exemple query +filter: +- query: + query_string: + query: "id: 2501 OR id: 5503" + +realert: + minutes: 0 + +# I only add the code content here. This way, it prevent to encode the entire description section. Only the log will be encoded and it will provide more visibility. +include: ["timestamp","name","computer"] +alert_text: "Alerts at {0} on the computer {1}.\n```" +alert_text_args: ["timestamp","computer"] + +# Needed +alert: +- discord +discord_webhook_url: "Your discord webhook url" + +# ----- Optional Section ----- + +discord_proxy: "proxy_address" + +# Must be in "" and must be valid emoji supported by discord. +discord_emoji_title: ":lock:" + +# Must be an hexadecimal value according to the exemple below +discord_embed_color: 0xE24D42 + +# This content will be displayed at the very end of your embed message. If you don't add one of these 2 lines, the footer will not be added. +discord_embed_footer: "Message sent by ElastAlert from your computer" +discord_embed_icon_url: "https://humancoders-formations.s3.amazonaws.com/uploads/course/logo/38/thumb_bigger_formation-elasticsearch.png" \ No newline at end of file From 3425e4a4e0643d56cb80ed8aedfdc63bbb7c58c3 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 18 Jan 2021 02:06:55 +0900 Subject: [PATCH 066/128] fix ruletypes.rst --- docs/source/ruletypes.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d7a19803..3750d6f4 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2297,9 +2297,7 @@ Optional: ``discord_emoji_title``: By default ElastAlert will use the ``:warning:`` emoji when posting to the channel. You can use a different emoji per ElastAlert rule. Any Apple emoji can be used, see http://emojipedia.org/apple/ . If slack_icon_url_override parameter is provided, emoji is ignored. -``discord_http_proxy``: By default ElastAlert will not use a network http proxy to send notifications to Discord. Set this option using hostname:port if you need to use a proxy. - -``discord_https_proxy``: By default ElastAlert will not use a network https proxy to send notifications to Discord. Set this option using hostname:port if you need to use a proxy. +``discord_proxy``: By default ElastAlert will not use a network proxy to send notifications to Discord. Set this option using hostname:port if you need to use a proxy. ``discord_proxy_login``: The Discord proxy auth username. From 62c15040a03ad7c94fa5f356d1c9807b800b28c7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 17 Jan 2021 18:38:16 -0500 Subject: [PATCH 067/128] Add CI workflow --- .github/workflows/alt_build_test.yml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/alt_build_test.yml diff --git a/.github/workflows/alt_build_test.yml b/.github/workflows/alt_build_test.yml new file mode 100644 index 00000000..7dab4f1c --- /dev/null +++ b/.github/workflows/alt_build_test.yml @@ -0,0 +1,30 @@ +# This is a basic workflow to help you get started with Actions + +name: alt_build_test + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ alt ] + pull_request: + branches: [ alt ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Build and run tests + run: make test-docker From 49710aaef83f9424efcda1355a807cdf48a79f47 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 17 Jan 2021 19:00:01 -0500 Subject: [PATCH 068/128] Update readme with fork information --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 7abb986e..94d20ec3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +# Fork of yelp/elastalert + +The original [yelp/elastalert](https://github.com/yelp/elastalert) repository has become mostly stale, with hundreds of open PRs and over 1000 open issues. The yelp team has acknowledged that they are winding down support of Elastalert. Consequently, it is difficult to merge fixes, dependency upgrades, and new features into Elastalert. Because of this, a fork of elastalert has been created. [jertel/elastalert](https://github.com/jertel/elastalert) will be an alternate repository for updates, until a new maintainer is appointed by the Yelp team and it's clear that the new maintainers are responding to PRs and issues. + +PRs are welcome, but must include tests, when possible. PRs will not be merged if they do not pass the automated CI workflows. + +The current status of the _alt_ branch CI workflow: + +![CI Workflow](https://github.com/jertel/elastalert/workflows/alt_build_test/badge.svg) + +If you're interested in a pre-built Docker image for either the official yelp/elastalert release, or for this fork, check out the [elastalert-docker](https://hub.docker.com/r/jertel/elastalert-docker) project on Docker Hub. + + +# Original README from yelp/elastalert follows... + Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) From 0bd519221842e1b3bdddf87f29f33c7fbee3548f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sat, 30 Jan 2021 01:57:28 +0900 Subject: [PATCH 069/128] Add Dingtalk --- README.md | 1 + docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 60 +++++++++++++++++++++++++ elastalert/alerts.py | 90 ++++++++++++++++++++++++++++++++++++++ elastalert/loaders.py | 3 +- elastalert/schema.yaml | 7 +++ 6 files changed, 161 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94d20ec3..2785cd73 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ Currently, we have built-in support for the following alert types: - TheHive - Zabbix - Discord +- Dingtalk Additional rule types and alerts can be easily imported or written. diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index f6683d55..37f424e5 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -56,6 +56,7 @@ Currently, we have support built in for these alert types: - TheHive - Zabbix - Discord +- Dingtalk Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 3750d6f4..bff211e0 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2308,3 +2308,63 @@ Optional: ``discord_embed_footer``: embed footer. ``discord_embed_icon_url``: You can provide icon_url to use custom image. Provide absolute address of the pciture. + +Dingtalk +~~~~~~~~ + +Dingtalk will send notification to a Line application. The body of the notification is formatted the same as with other alerters. + +Required: + +``dingtalk_access_token``: Dingtalk access token. + +``dingtalk_msgtype``: Dingtalk msgtype. ``text``, ``markdown``, ``single_action_card``, ``action_card``. + +dingtalk_msgtype single_action_card Required: + +``dingtalk_single_title``: The title of a single button.. + +``dingtalk_single_url``: Jump link for a single button. + +dingtalk_msgtype action_card Required: + +``dingtalk_btns``: Button. + +dingtalk_msgtype action_card Optional: + +``dingtalk_btn_orientation``: "0": Buttons are arranged vertically "1": Buttons are arranged horizontally. + +Example msgtype : text:: + + alert: + - dingtalk + dingtalk_access_token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + dingtalk_msgtype: 'text' + + +Example msgtype : markdown:: + + alert: + - dingtalk + dingtalk_access_token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + dingtalk_msgtype: 'markdown' + + +Example msgtype : single_action_card:: + + alert: + - dingtalk + dingtalk_access_token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + dingtalk_msgtype: 'single_action_card' + dingtalk_single_title: 'test3' + dingtalk_single_url: 'https://xxxx.xxx' + + +Example msgtype : action_card:: + + alert: + - dingtalk + dingtalk_access_token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + dingtalk_msgtype: 'action_card' + dingtalk_btn_orientation: '0' + dingtalk_btns: [{'title': 'a', 'actionURL': 'https://xxxx1.xxx'}, {'title': 'b', 'actionURL': 'https://xxxx2.xxx'}] diff --git a/elastalert/alerts.py b/elastalert/alerts.py index e44cdd68..34c7d0fc 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2072,3 +2072,93 @@ def alert(self, matches): def get_info(self): return {'type': 'discord', 'discord_webhook_url': self.discord_webhook_url} + + +class DingTalkAlerter(Alerter): + """ Creates a DingTalk room message for each alert """ + required_options = frozenset(['dingtalk_access_token', 'dingtalk_msgtype']) + + def __init__(self, rule): + super(DingTalkAlerter, self).__init__(rule) + self.dingtalk_access_token = self.rule.get('dingtalk_access_token') + self.dingtalk_webhook_url = 'https://oapi.dingtalk.com/robot/send?access_token=%s' % (self.dingtalk_access_token) + self.dingtalk_msgtype = self.rule.get('dingtalk_msgtype') + self.dingtalk_single_title = self.rule.get('dingtalk_single_title', 'elastalert') + self.dingtalk_single_url = self.rule.get('dingtalk_single_url', '') + self.dingtalk_btn_orientation = self.rule.get('dingtalk_btn_orientation', '') + self.dingtalk_btns = self.rule.get('dingtalk_btns', []) + self.dingtalk_proxy = self.rule.get('dingtalk_proxy', None) + self.dingtalk_proxy_login = self.rule.get('dingtalk_proxy_login', None) + self.dingtalk_proxy_password = self.rule.get('dingtalk_proxy_pass', None) + + def format_body(self, body): + return body.encode('utf8') + + def alert(self, matches): + title = self.create_title(matches) + body = self.create_alert_body(matches) + + proxies = {'https': self.dingtalk_proxy} if self.dingtalk_proxy else None + auth = HTTPProxyAuth(self.dingtalk_proxy_login, self.dingtalk_proxy_password) if self.dingtalk_proxy_login else None + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json;charset=utf-8' + } + + if self.dingtalk_msgtype == 'text': + # text + payload = { + 'msgtype': self.dingtalk_msgtype, + 'text': { + 'content': body + } + } + elif self.dingtalk_msgtype == 'markdown': + # markdown + payload = { + 'msgtype': self.dingtalk_msgtype, + 'markdown': { + 'title': title, + 'text': body + } + } + elif self.dingtalk_msgtype == 'single_action_card': + # singleActionCard + payload = { + 'msgtype': 'actionCard', + 'actionCard': { + 'title': title, + 'text': body, + 'singleTitle': self.dingtalk_single_title, + 'singleURL': self.dingtalk_single_url + } + } + elif self.dingtalk_msgtype == 'action_card': + # actionCard + payload = { + 'msgtype': 'actionCard', + 'actionCard': { + 'title': title, + 'text': body + } + } + if self.dingtalk_btn_orientation != '': + payload['actionCard']['btnOrientation'] = self.dingtalk_btn_orientation + if self.dingtalk_btns: + payload['actionCard']['btns'] = self.dingtalk_btns + + try: + response = requests.post(self.dingtalk_webhook_url, data=json.dumps(payload, + cls=DateTimeEncoder), headers=headers, proxies=proxies, auth=auth) + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to dingtalk: %s" % e) + + elastalert_logger.info("Trigger sent to dingtalk") + + def get_info(self): + return { + "type": "dingtalk", + "dingtalk_webhook_url": self.dingtalk_webhook_url + } diff --git a/elastalert/loaders.py b/elastalert/loaders.py index f2b1701b..766d0fe2 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -81,7 +81,8 @@ class RulesLoader(object): 'linenotify': alerts.LineNotifyAlerter, 'hivealerter': alerts.HiveAlerter, 'zabbix': ZabbixAlerter, - 'discord': alerts.DiscordAlerter + 'discord': alerts.DiscordAlerter, + 'dingtalk': alerts.DingTalkAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index d249b327..0fd37760 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -374,3 +374,10 @@ properties: ## Discord discord_webhook_url: {type: string} + + ### Dingtalk + dingtalk_access_token: {type: string} + dingtalk_msgtype: {type: string} + dingtalk_single_title: {type: string} + dingtalk_single_url: {type: string} + dingtalk_btn_orientation: {type: string} \ No newline at end of file From f61bb912e4c1895b668f0ab9e32a6744059cac74 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 31 Jan 2021 00:00:45 +0900 Subject: [PATCH 070/128] Add Chatwork Alerter / Fix a typo --- README.md | 1 + docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 22 +++++++++++++++++++-- elastalert/alerts.py | 39 ++++++++++++++++++++++++++++++++++++++ elastalert/loaders.py | 3 ++- elastalert/schema.yaml | 6 +++++- 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2785cd73..fb284142 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Currently, we have built-in support for the following alert types: - Zabbix - Discord - Dingtalk +- Chatwork Additional rule types and alerts can be easily imported or written. diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 37f424e5..17dbf5b0 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -57,6 +57,7 @@ Currently, we have support built in for these alert types: - Zabbix - Discord - Dingtalk +- Chatwork Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0a654dae..4d4ec963 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2292,7 +2292,7 @@ Required: Discord ~~~~~~~ -Discord will send notification to a Line application. The body of the notification is formatted the same as with other alerters. +Discord will send notification to a Discord application. The body of the notification is formatted the same as with other alerters. Required: @@ -2317,7 +2317,7 @@ Optional: Dingtalk ~~~~~~~~ -Dingtalk will send notification to a Line application. The body of the notification is formatted the same as with other alerters. +Dingtalk will send notification to a Dingtalk application. The body of the notification is formatted the same as with other alerters. Required: @@ -2373,3 +2373,21 @@ Example msgtype : action_card:: dingtalk_msgtype: 'action_card' dingtalk_btn_orientation: '0' dingtalk_btns: [{'title': 'a', 'actionURL': 'https://xxxx1.xxx'}, {'title': 'b', 'actionURL': 'https://xxxx2.xxx'}] + +Chatwork +~~~~~~~~ + +Chatwork will send notification to a Chatwork application. The body of the notification is formatted the same as with other alerters. + +Required: + +``chatwork_apikey``: ChatWork API KEY. + +``chatwork_room_id``: The ID of the room you are talking to in Chatwork. How to find the room ID is the part of the number after "rid" at the end of the URL of the browser. + +Example usage:: + + alert: + - chatwork + chatwork_apikey: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + chatwork_room_id: 'xxxxxxxxx' diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d0b735e5..eb29e94e 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2191,3 +2191,42 @@ def get_info(self): "type": "dingtalk", "dingtalk_webhook_url": self.dingtalk_webhook_url } + + +class ChatworkAlerter(Alerter): + """ Creates a Chatwork room message for each alert """ + required_options = frozenset(['chatwork_apikey', 'chatwork_room_id']) + + def __init__(self, rule): + super(ChatworkAlerter, self).__init__(rule) + self.chatwork_apikey = self.rule.get('chatwork_apikey') + self.chatwork_room_id = self.rule.get('chatwork_room_id') + self.url = 'https://api.chatwork.com/v2/rooms/%s/messages' % (self.chatwork_room_id) + self.chatwork_proxy = self.rule.get('chatwork_proxy', None) + self.chatwork_proxy_login = self.rule.get('chatwork_proxy_login', None) + self.chatwork_proxy_pass = self.rule.get('chatwork_proxy_pass', None) + + def alert(self, matches): + body = self.create_alert_body(matches) + + headers = {'X-ChatWorkToken': self.chatwork_apikey} + # set https proxy, if it was provided + proxies = {'https': self.chatwork_proxy} if self.chatwork_proxy else None + auth = HTTPProxyAuth(self.chatwork_proxy_login, self.chatwork_proxy_pass) if self.chatwork_proxy_login else None + params = {'body': body} + + try: + response = requests.post(self.url, params=params, headers=headers, proxies=proxies, auth=auth) + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Chattwork: %s. Details: %s" % (e, "" if e.response is None else e.response.text)) + + elastalert_logger.info( + "Alert sent to Chatwork room %s" % self.chatwork_room_id) + + def get_info(self): + return { + "type": "chatwork", + "chatwork_room_id": self.chatwork_room_id + } diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 766d0fe2..36aa7451 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -82,7 +82,8 @@ class RulesLoader(object): 'hivealerter': alerts.HiveAlerter, 'zabbix': ZabbixAlerter, 'discord': alerts.DiscordAlerter, - 'dingtalk': alerts.DingTalkAlerter + 'dingtalk': alerts.DingTalkAlerter, + 'chatwork': alerts.ChatworkAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index ca0dfd56..aab13c01 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -382,4 +382,8 @@ properties: dingtalk_msgtype: {type: string} dingtalk_single_title: {type: string} dingtalk_single_url: {type: string} - dingtalk_btn_orientation: {type: string} \ No newline at end of file + dingtalk_btn_orientation: {type: string} + + ### Chatwork + chatwork_apikey: {type: string} + chatwork_room_id: {type: string} \ No newline at end of file From 6b6af45a02e4f3314e67f3dee7d753a646546f1d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 12 Feb 2021 22:54:31 +0900 Subject: [PATCH 071/128] Update requirements-dev.txt --- requirements-dev.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d15887c0..f99a6f96 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,10 @@ -r requirements.txt -coverage==4.5.4 +coverage==5.4 flake8 +pluggy>=0.12.0 pre-commit -pylint<1.4 -pytest<3.3.0 +pylint<2.7 +pytest<3.7.0 setuptools sphinx_rtd_theme -tox==3.20.1 +tox==3.21.4 From 87449308778a613e234b355e00437aa40bb6c0c8 Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Sun, 21 Feb 2021 18:11:56 +0300 Subject: [PATCH 072/128] Creates a new rule config parameter that "use_local_time_for_query". It provide that use local time when query in "run_query". So, filters able to use local time can create. Indices able to have @timestap field with different timezone from "UTC" can be. --- elastalert/elastalert.py | 5 +++++ elastalert/loaders.py | 1 + elastalert/util.py | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 7e3f24c1..43d249ff 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -53,6 +53,7 @@ from .util import ts_now from .util import ts_to_dt from .util import unix_to_dt +from .util import ts_utc_to_local class ElastAlerter(object): @@ -629,6 +630,10 @@ def run_query(self, rule, start=None, end=None, scroll=False): if end is None: end = ts_now() + if rule.get('use_local_time_for_query'): + start = ts_utc_to_local(start) + end = ts_utc_to_local(end) + # Reset hit counter and query rule_inst = rule['type'] rule['scrolling_cycle'] = rule.get('scrolling_cycle', 0) + 1 diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 36aa7451..e4983364 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -283,6 +283,7 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('use_local_time', True) rule.setdefault('description', "") rule.setdefault('jinja_root_name', "_data") + rule.setdefault('use_local_time_for_query', False) # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() diff --git a/elastalert/util.py b/elastalert/util.py index 3e9c9f66..90021ac2 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -185,6 +185,9 @@ def dt_to_ts_with_format(dt, ts_format): def ts_now(): return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) +def ts_utc_to_local(ts): + """Convert utc time to local time.""" + return ts.astimezone(dateutil.tz.tzlocal()) def inc_ts(timestamp, milliseconds=1): """Increment a timestamp by milliseconds.""" @@ -202,7 +205,7 @@ def pretty_ts(timestamp, tz=True): dt = ts_to_dt(timestamp) if tz: dt = dt.astimezone(dateutil.tz.tzlocal()) - return dt.strftime('%Y-%m-%d %H:%M %Z') + return dt.strftime('%Y-%m-%d %H:%M %z') def ts_add(ts, td): From 0fa2e9dfb27f028712659be615785f7f74d083cd Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Sun, 21 Feb 2021 18:30:07 +0300 Subject: [PATCH 073/128] Adds information about the "use_local_time_for_query" parameter. --- docs/source/ruletypes.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4d4ec963..f01429e3 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -102,6 +102,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``scan_entire_timeframe`` (bool, default False) | | +--------------------------------------------------------------+ | +| ``use_local_time_for_query`` (bool, default False) | | ++--------------------------------------------------------------+ | | ``import`` (string) | | | | | | IGNORED IF ``use_count_query`` or ``use_terms_query`` is true| | @@ -695,6 +697,11 @@ scan the same range again, triggering duplicate alerts. Some rules and alerts require additional options, which also go in the top level of the rule configuration file. +use_local_time_for_query +^^^^^^^^^^^^^^^^^^^^^ + +``use_local_time_for_query``: Whether to convert UTC time to the local time zone in rule queries. +If false, start and end time of query will be used UTC. (Optional, boolean, default false) .. _testing : From 0dd8cd8d1b83b7442e84dd488feba85a77272937 Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Sun, 21 Feb 2021 18:46:52 +0300 Subject: [PATCH 074/128] Fixes the documentation error. --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f01429e3..a9919973 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -698,7 +698,7 @@ scan the same range again, triggering duplicate alerts. Some rules and alerts require additional options, which also go in the top level of the rule configuration file. use_local_time_for_query -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^ ``use_local_time_for_query``: Whether to convert UTC time to the local time zone in rule queries. If false, start and end time of query will be used UTC. (Optional, boolean, default false) From b27a50ee6dae55ca6388a0e087da60842f912ebf Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Sun, 21 Feb 2021 18:52:59 +0300 Subject: [PATCH 075/128] Fixes styling problems. --- elastalert/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/util.py b/elastalert/util.py index 90021ac2..39fadd35 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -185,10 +185,12 @@ def dt_to_ts_with_format(dt, ts_format): def ts_now(): return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) + def ts_utc_to_local(ts): """Convert utc time to local time.""" return ts.astimezone(dateutil.tz.tzlocal()) + def inc_ts(timestamp, milliseconds=1): """Increment a timestamp by milliseconds.""" dt = ts_to_dt(timestamp) From 55562c2fc7f936c5e333a0bf6c413e7c9e2b240a Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Tue, 23 Feb 2021 09:38:47 +0300 Subject: [PATCH 076/128] Rename "use_local_time_for_query" to "query_timezone". If "query_timezone" set, elasticsearch query start and end date will convert from UTC to specified time zone. --- docs/source/ruletypes.rst | 12 +++++++----- elastalert/elastalert.py | 9 +++++---- elastalert/loaders.py | 2 +- elastalert/util.py | 4 ++-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a9919973..e2efd1e1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -102,7 +102,7 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``scan_entire_timeframe`` (bool, default False) | | +--------------------------------------------------------------+ | -| ``use_local_time_for_query`` (bool, default False) | | +| ``query_timezone`` (string, default empty string) | | +--------------------------------------------------------------+ | | ``import`` (string) | | | | | @@ -697,11 +697,13 @@ scan the same range again, triggering duplicate alerts. Some rules and alerts require additional options, which also go in the top level of the rule configuration file. -use_local_time_for_query -^^^^^^^^^^^^^^^^^^^^^^^^ +query_timezone +^^^^^^^^^^^^^^ + +``query_timezone``: Whether to convert UTC time to the specified time zone in rule queries. +If not set, start and end time of query will be used UTC. (Optional, string, default empty string) -``use_local_time_for_query``: Whether to convert UTC time to the local time zone in rule queries. -If false, start and end time of query will be used UTC. (Optional, boolean, default false) +Example value : query_timezone: "Europe/Istanbul" .. _testing : diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 43d249ff..e689619f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -53,7 +53,7 @@ from .util import ts_now from .util import ts_to_dt from .util import unix_to_dt -from .util import ts_utc_to_local +from .util import ts_utc_to_tz class ElastAlerter(object): @@ -630,9 +630,10 @@ def run_query(self, rule, start=None, end=None, scroll=False): if end is None: end = ts_now() - if rule.get('use_local_time_for_query'): - start = ts_utc_to_local(start) - end = ts_utc_to_local(end) + if rule.get('query_timezone') != "": + elastalert_logger.info("Query start and end time converting UTC to query_timezone : {}".format(rule.get('query_timezone'))) + start = ts_utc_to_tz(start, rule.get('query_timezone')) + end = ts_utc_to_tz(end, rule.get('query_timezone')) # Reset hit counter and query rule_inst = rule['type'] diff --git a/elastalert/loaders.py b/elastalert/loaders.py index e4983364..bee5d475 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -283,7 +283,7 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('use_local_time', True) rule.setdefault('description', "") rule.setdefault('jinja_root_name', "_data") - rule.setdefault('use_local_time_for_query', False) + rule.setdefault('query_timezone', "") # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() diff --git a/elastalert/util.py b/elastalert/util.py index 39fadd35..f3f7dde1 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -186,9 +186,9 @@ def ts_now(): return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) -def ts_utc_to_local(ts): +def ts_utc_to_tz(ts,tz_name): """Convert utc time to local time.""" - return ts.astimezone(dateutil.tz.tzlocal()) + return ts.astimezone(dateutil.tz.gettz(tz_name)) def inc_ts(timestamp, milliseconds=1): From b5a55edb5e9139924df89f85a176c0fab7b28b6a Mon Sep 17 00:00:00 2001 From: Dilaver Demirel Date: Tue, 23 Feb 2021 09:43:38 +0300 Subject: [PATCH 077/128] Fix styling problem. --- elastalert/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/util.py b/elastalert/util.py index f3f7dde1..658c6bf4 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -186,7 +186,7 @@ def ts_now(): return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) -def ts_utc_to_tz(ts,tz_name): +def ts_utc_to_tz(ts, tz_name): """Convert utc time to local time.""" return ts.astimezone(dateutil.tz.gettz(tz_name)) From 6f35f8205d615858e8a5bc904738e886ca71636f Mon Sep 17 00:00:00 2001 From: Mostafa Hussein Date: Wed, 24 Feb 2021 22:11:26 +0200 Subject: [PATCH 078/128] allow alerta_severity to be set dynamically --- elastalert/alerts.py | 2 +- elastalert/schema.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index eb29e94e..71071d31 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1856,7 +1856,7 @@ def get_json_payload(self, match): alerta_payload_dict = { 'resource': resolve_string(self.resource, match, self.missing_text), - 'severity': self.severity, + 'severity': resolve_string(self.severity, match), 'timeout': self.timeout, 'createTime': createTime, 'type': self.type, diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index aab13c01..516338e3 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -343,7 +343,7 @@ properties: ### Alerta alerta_api_url: {type: string} alerta_api_key: {type: string} - alerta_severity: {enum: [unknown, security, debug, informational, ok, normal, cleared, indeterminate, warning, minor, major, critical]} + alerta_severity: {type: string} alerta_resource: {type: string} # Python format string alerta_environment: {type: string} # Python format string alerta_origin: {type: string} # Python format string @@ -386,4 +386,4 @@ properties: ### Chatwork chatwork_apikey: {type: string} - chatwork_room_id: {type: string} \ No newline at end of file + chatwork_room_id: {type: string} From d54ddf0258add37ed14d0a5b0c0762e24902a1b1 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 26 Feb 2021 02:08:38 +0900 Subject: [PATCH 079/128] Kibana Discover support kibana 7.11 --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index e2efd1e1..74b3bb0d 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -555,9 +555,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11` -``kibana_discover_version: '7.10'`` +``kibana_discover_version: '7.11'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index ed8b681d..0cbbc211 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -15,7 +15,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 516338e3..65f3dc3e 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -219,7 +219,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index a508c25d..30191b89 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.10', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.10', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.10', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.10', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.10', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From 6c142ce13d371717c17585bf29fb2d6260b74f52 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Fri, 26 Feb 2021 02:18:03 +0900 Subject: [PATCH 080/128] Update pylint and tox pylint from <2.7 to <2.8 tox from 3.21.4 to 3.22.0 --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f99a6f96..6b0642e5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,8 +3,8 @@ coverage==5.4 flake8 pluggy>=0.12.0 pre-commit -pylint<2.7 +pylint<2.8 pytest<3.7.0 setuptools sphinx_rtd_theme -tox==3.21.4 +tox==3.22.0 From a78620518d2e89eb6e76356d6a2da204107dbaa7 Mon Sep 17 00:00:00 2001 From: feroz Date: Fri, 26 Feb 2021 15:02:12 +0000 Subject: [PATCH 081/128] Add a default description for TheHive alerts if one isn't provided Use the default alert body created by ElastAlert, in line with the other alerters --- elastalert/alerts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 71071d31..c70128c4 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1992,11 +1992,12 @@ def alert(self, matches): alert_config = { 'artifacts': artifacts, - 'sourceRef': str(uuid.uuid4())[0:6], - 'customFields': {}, 'caseTemplate': None, + 'customFields': {}, + 'date': int(time.time()) * 1000, + 'description': self.create_alert_body(matches), + 'sourceRef': str(uuid.uuid4())[0:6], 'title': '{rule[index]}_{rule[name]}'.format(**context), - 'date': int(time.time()) * 1000 } alert_config.update(self.rule.get('hive_alert_config', {})) custom_fields = {} From 5ce0a44838e056ccf1fd7d6297eef02aa12acb14 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 28 Feb 2021 12:44:15 +0900 Subject: [PATCH 082/128] added docs slack slack_parse_override slack_text_string --- docs/source/ruletypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 74b3bb0d..b2c64b24 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1792,6 +1792,10 @@ Provide absolute address of the pciture. ``slack_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good' or 'warning' colors. +``slack_parse_override``: By default the notification message is escaped 'none'. You can also use 'full'. + +``slack_text_string``: Notification message you want to add. + ``slack_proxy``: By default ElastAlert will not use a network proxy to send notifications to Slack. Set this option using ``hostname:port`` if you need to use a proxy. ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. From 450ebe4a530b269c849380d2b5d297444569ec73 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 28 Feb 2021 19:37:57 +0900 Subject: [PATCH 083/128] add docs jira_assignee --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 74b3bb0d..8a850b22 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1566,6 +1566,8 @@ For an example JIRA account file, see ``example_rules/jira_acct.yaml``. The acco Optional: +``jira_assignee``: Assigns an issue to a user. + ``jira_component``: The name of the component or components to set the ticket to. This can be a single string or a list of strings. This is provided for backwards compatibility and will eventually be deprecated. It is preferable to use the plural ``jira_components`` instead. ``jira_components``: The name of the component or components to set the ticket to. This can be a single string or a list of strings. From c5f1f1c0287ade43750fe7f3252b1dd8fc82ec7a Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Mon, 1 Mar 2021 18:04:04 +0000 Subject: [PATCH 084/128] Update README to point to new docs, deprecate old README - Update the README to point to the new ReadTheDocs link - Move the old Elastalert README to a new file to reduce confusion - Add some more text to the new README that covers the essential points of the old README --- README-old.md | 373 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 392 ++++---------------------------------------------- 2 files changed, 400 insertions(+), 365 deletions(-) create mode 100644 README-old.md diff --git a/README-old.md b/README-old.md new file mode 100644 index 00000000..bbdf39d9 --- /dev/null +++ b/README-old.md @@ -0,0 +1,373 @@ +# Deprecated ElastAlert README + +*The documentation below refers to a previous ElastAlert version. For the latest version, see [here][0]* + +Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. + +[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) +[![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org). +### Easy & Flexible Alerting With Elasticsearch + +ElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. + +ElastAlert works with all versions of Elasticsearch. + +At Yelp, we use Elasticsearch, Logstash and Kibana for managing our ever increasing amount of data and logs. +Kibana is great for visualizing and querying data, but we quickly realized that it needed a companion tool for alerting +on inconsistencies in our data. Out of this need, ElastAlert was created. + +If you have data being written into Elasticsearch in near real time and want to be alerted when that data matches certain patterns, ElastAlert is the tool for you. If you can see it in Kibana, ElastAlert can alert on it. + +## Overview + +We designed ElastAlert to be reliable, highly modular, and easy to set up and configure. + +It works by combining Elasticsearch with two types of components, rule types and alerts. +Elasticsearch is periodically queried and the data is passed to the rule type, which determines when +a match is found. When a match occurs, it is given to one or more alerts, which take action based on the match. + +This is configured by a set of rules, each of which defines a query, a rule type, and a set of alerts. + +Several rule types with common monitoring paradigms are included with ElastAlert: + +- Match where there are at least X events in Y time" (``frequency`` type) +- Match when the rate of events increases or decreases" (``spike`` type) +- Match when there are less than X events in Y time" (``flatline`` type) +- Match when a certain field matches a blacklist/whitelist" (``blacklist`` and ``whitelist`` type) +- Match on any event matching a given filter" (``any`` type) +- Match when a field has two different values within some time" (``change`` type) +- Match when a never before seen term appears in a field" (``new_term`` type) +- Match when the number of unique values for a field is above or below a threshold (``cardinality`` type) + +Currently, we have built-in support for the following alert types: + +- Command +- Email +- JIRA +- OpsGenie +- AWS SNS +- MS Teams +- Slack +- Mattermost +- Telegram +- GoogleChat +- PagerDuty +- PagerTree +- Exotel +- Twilio +- Splunk On-Call (Formerly VictorOps) +- Gitter +- ServiceNow +- Debug +- Stomp +- Alerta +- HTTP POST +- Line Notify +- TheHive +- Zabbix +- Discord +- Dingtalk +- Chatwork + +Additional rule types and alerts can be easily imported or written. + +In addition to this basic usage, there are many other features that make alerts more useful: + +- Alerts link to Kibana dashboards +- Aggregate counts for arbitrary fields +- Combine alerts into periodic reports +- Separate alerts by using a unique key field +- Intercept and enhance match data + +To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). + +## Running ElastAlert +You can either install the latest released version of ElastAlert using pip: + +```pip install elastalert``` + +or you can clone the ElastAlert repository for the most recent changes: + +```git clone https://github.com/Yelp/elastalert.git``` + +Install the module: + +```pip install "setuptools>=11.3"``` + +```python setup.py install``` + +The following invocation can be used to run ElastAlert after installing + +``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` + +``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. + +``--verbose`` will print additional information without suppressing alerts. Not compatible with `--debug.` + +``--start`` will begin querying at the given timestamp. By default, ElastAlert will begin querying from the present. +Timestamp format is ``YYYY-MM-DDTHH-MM-SS[-/+HH:MM]`` (Note the T between date and hour). +Eg: ``--start 2014-09-26T12:00:00`` (UTC) or ``--start 2014-10-01T07:30:00-05:00`` + +``--end`` will cause ElastAlert to stop querying at the given timestamp. By default, ElastAlert will continue +to query indefinitely. + +``--rule`` will allow you to run only one rule. It must still be in the rules folder. +Eg: ``--rule this_rule.yaml`` + +``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory. + +## Third Party Tools And Extras +### Kibana plugin +![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) +Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). + +### Docker +A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. + +```bash +git clone https://github.com/bitsensor/elastalert.git; cd elastalert +docker run -d -p 3030:3030 -p 3333:3333 \ + -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ + -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ + -v `pwd`/rules:/opt/elastalert/rules \ + -v `pwd`/rule_templates:/opt/elastalert/rule_templates \ + --net="host" \ + --name elastalert bitsensor/elastalert:latest +``` + +### ElastAlert-Docker + +This Dockerfile will build a Docker image for Elastalert. This image is compatible with the accompanying Helm chart for Kubernetes. + +https://github.com/jertel/elastalert-docker + +### ElastAlert uses ElastAlert Helm Chart almost as it is *Use original Docker Image to fix bugs + +[ElastAlert Server Helm Chart](https://github.com/daichi703n/elastalert-helm) + +[Praeco Helm Chart](https://github.com/daichi703n/praeco-helm) + +[Installing Praeco (ElastAlert GUI) into Kubernetes with Helm](https://en-designetwork.daichi703n.com/entry/2020/02/24/praeco-helm-kubernetes) + +### Praeco +![Praeco screenshot](https://user-images.githubusercontent.com/611996/47752071-7c4a9080-dc61-11e8-8ccf-2196f13429b2.png) +[Praeco](https://github.com/johnsusek/praeco) is a free open source GUI for ElastAlert. + +## Documentation + +Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). + +To build a html version of the docs locally + +``` +pip install sphinx_rtd_theme sphinx +cd docs +make html +``` + +View in browser at build/html/index.html + +## Configuration + +See config.yaml.example for details on configuration. + +## Example rules + +Examples of different types of rules can be found in example_rules/. + +- ``example_spike.yaml`` is an example of the "spike" rule type, which allows you to alert when the rate of events, averaged over a time period, +increases by a given factor. This example will send an email alert when there are 3 times more events matching a filter occurring within the +last 2 hours than the number of events in the previous 2 hours. + +- ``example_frequency.yaml`` is an example of the "frequency" rule type, which will alert when there are a given number of events occuring +within a time period. This example will send an email when 50 documents matching a given filter occur within a 4 hour timeframe. + +- ``example_change.yaml`` is an example of the "change" rule type, which will alert when a certain field in two documents changes. In this example, +the alert email is sent when two documents with the same 'username' field but a different value of the 'country_name' field occur within 24 hours +of each other. + +- ``example_new_term.yaml`` is an example of the "new term" rule type, which alerts when a new value appears in a field or fields. In this example, +an email is sent when a new value of ("username", "computer") is encountered in example login logs. + +## Frequently Asked Questions + +### My rule is not getting any hits? + +So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like + +``` +filter: +- query: + query_string: + query: "foo: bar AND baz: abc*" +``` +If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example, + +``` +filter: +- term: + foo: "Test Document" +``` + +will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening. + +### I got hits, why didn't I get an alert? + +If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types. + +If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs. + +If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" as the "From:" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this. + +### Why did I only get one alert when I expected to get several? + +There is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use + +``` +realert: + minutes: 0 +``` + +You can of course set it higher as well. + +### How can I prevent duplicate alerts? + +By setting ``realert``, you will prevent the same rule from alerting twice in an amount of time. + +``` +realert: + days: 1 +``` + +You can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use + +``` +realert: + hours: 8 +query_key: user +``` + +Note that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use + +``` +query_key: [error, hostname] +``` + +Internally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``. + +The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory. + +### How can I change what's in the alert? + +You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: + +With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. + +``` +alert_text_type: alert_text_jinja + +alert_text: | + Alert triggered! *({{num_hits}} Matches!)* + Something happened with {{username}} ({{email}}) + {{description|truncate}} + +``` + +> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. +> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. + +With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example + +``` +alert_text: "Something happened with {0} at {1}" +alert_text_type: alert_text_only +alert_text_args: ["username", "@timestamp"] +``` + +You can also limit the alert to only containing certain fields from the document by using ``include``. + +``` +include: ["ip_address", "hostname", "status"] +``` + +### My alert only contains data for one event, how can I see more? + +If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have + +``` +top_count_keys: ["ip_address", "status"] +``` + +and 10 documents matched your alert, it may contain something like + +``` +ip_address: +127.0.0.1: 7 +10.0.0.1: 2 +192.168.0.1: 1 + +status: +200: 9 +500: 1 +``` + +### How can I make the alert come at a certain time? + +The ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using + +``` +aggregation: + schedule: '2 4 * * mon,fri' +``` + +### I have lots of documents and it's really slow, how can I speed it up? + +There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` + +``` +index: logstash-%Y.%m +use_strftime_index: true +``` + +Another thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes. + +``` +buffer_time: + minutes: 5 +``` + +By default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp. + +If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types. + +### Can I perform aggregations? + +The only aggregation supported currently is a terms aggregation, by setting ``use_terms_query``. + +### I'm not using @timestamp, what do I do? + +You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents. + +### I'm using flatline but I don't see any alerts + +When using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them. + +### How can I get a "resolve" event? + +ElastAlert does not currently support stateful alerts or resolve events. + +### Can I set a warning threshold? + +Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. + +## License + +ElastAlert is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + +### Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). + +### Questions? Drop by #elastalert on Freenode IRC. + +[0]: https://github.com/jertel/elastalert/blob/alt/README.md diff --git a/README.md b/README.md index fb284142..694f03eb 100644 --- a/README.md +++ b/README.md @@ -1,382 +1,44 @@ # Fork of yelp/elastalert -The original [yelp/elastalert](https://github.com/yelp/elastalert) repository has become mostly stale, with hundreds of open PRs and over 1000 open issues. The yelp team has acknowledged that they are winding down support of Elastalert. Consequently, it is difficult to merge fixes, dependency upgrades, and new features into Elastalert. Because of this, a fork of elastalert has been created. [jertel/elastalert](https://github.com/jertel/elastalert) will be an alternate repository for updates, until a new maintainer is appointed by the Yelp team and it's clear that the new maintainers are responding to PRs and issues. - -PRs are welcome, but must include tests, when possible. PRs will not be merged if they do not pass the automated CI workflows. - -The current status of the _alt_ branch CI workflow: - -![CI Workflow](https://github.com/jertel/elastalert/workflows/alt_build_test/badge.svg) - -If you're interested in a pre-built Docker image for either the official yelp/elastalert release, or for this fork, check out the [elastalert-docker](https://hub.docker.com/r/jertel/elastalert-docker) project on Docker Hub. - - -# Original README from yelp/elastalert follows... - -Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. - -[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) -[![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org). -### Easy & Flexible Alerting With Elasticsearch - -ElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. - -ElastAlert works with all versions of Elasticsearch. - -At Yelp, we use Elasticsearch, Logstash and Kibana for managing our ever increasing amount of data and logs. -Kibana is great for visualizing and querying data, but we quickly realized that it needed a companion tool for alerting -on inconsistencies in our data. Out of this need, ElastAlert was created. - -If you have data being written into Elasticsearch in near real time and want to be alerted when that data matches certain patterns, ElastAlert is the tool for you. If you can see it in Kibana, ElastAlert can alert on it. - -## Overview - -We designed ElastAlert to be reliable, highly modular, and easy to set up and configure. - -It works by combining Elasticsearch with two types of components, rule types and alerts. -Elasticsearch is periodically queried and the data is passed to the rule type, which determines when -a match is found. When a match occurs, it is given to one or more alerts, which take action based on the match. - -This is configured by a set of rules, each of which defines a query, a rule type, and a set of alerts. - -Several rule types with common monitoring paradigms are included with ElastAlert: - -- Match where there are at least X events in Y time" (``frequency`` type) -- Match when the rate of events increases or decreases" (``spike`` type) -- Match when there are less than X events in Y time" (``flatline`` type) -- Match when a certain field matches a blacklist/whitelist" (``blacklist`` and ``whitelist`` type) -- Match on any event matching a given filter" (``any`` type) -- Match when a field has two different values within some time" (``change`` type) -- Match when a never before seen term appears in a field" (``new_term`` type) -- Match when the number of unique values for a field is above or below a threshold (``cardinality`` type) - -Currently, we have built-in support for the following alert types: - -- Command -- Email -- JIRA -- OpsGenie -- AWS SNS -- MS Teams -- Slack -- Mattermost -- Telegram -- GoogleChat -- PagerDuty -- PagerTree -- Exotel -- Twilio -- Splunk On-Call (Formerly VictorOps) -- Gitter -- ServiceNow -- Debug -- Stomp -- Alerta -- HTTP POST -- Line Notify -- TheHive -- Zabbix -- Discord -- Dingtalk -- Chatwork - -Additional rule types and alerts can be easily imported or written. - -In addition to this basic usage, there are many other features that make alerts more useful: - -- Alerts link to Kibana dashboards -- Aggregate counts for arbitrary fields -- Combine alerts into periodic reports -- Separate alerts by using a unique key field -- Intercept and enhance match data - -To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). - -## Running ElastAlert -You can either install the latest released version of ElastAlert using pip: - -```pip install elastalert``` - -or you can clone the ElastAlert repository for the most recent changes: - -```git clone https://github.com/Yelp/elastalert.git``` - -Install the module: - -```pip install "setuptools>=11.3"``` - -```python setup.py install``` - -The following invocation can be used to run ElastAlert after installing - -``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` - -``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. - -``--verbose`` will print additional information without suppressing alerts. Not compatible with `--debug.` - -``--start`` will begin querying at the given timestamp. By default, ElastAlert will begin querying from the present. -Timestamp format is ``YYYY-MM-DDTHH-MM-SS[-/+HH:MM]`` (Note the T between date and hour). -Eg: ``--start 2014-09-26T12:00:00`` (UTC) or ``--start 2014-10-01T07:30:00-05:00`` - -``--end`` will cause ElastAlert to stop querying at the given timestamp. By default, ElastAlert will continue -to query indefinitely. - -``--rule`` will allow you to run only one rule. It must still be in the rules folder. -Eg: ``--rule this_rule.yaml`` - -``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory. - -## Third Party Tools And Extras -### Kibana plugin -![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) -Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). - -### Docker -A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. - -```bash -git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 -p 3333:3333 \ - -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ - -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ - -v `pwd`/rules:/opt/elastalert/rules \ - -v `pwd`/rule_templates:/opt/elastalert/rule_templates \ - --net="host" \ - --name elastalert bitsensor/elastalert:latest -``` - -### ElastAlert-Docker - -This Dockerfile will build a Docker image for Elastalert. This image is compatible with the accompanying Helm chart for Kubernetes. - -https://github.com/jertel/elastalert-docker - -### ElastAlert uses ElastAlert Helm Chart almost as it is *Use original Docker Image to fix bugs - -[ElastAlert Server Helm Chart](https://github.com/daichi703n/elastalert-helm) - -[Praeco Helm Chart](https://github.com/daichi703n/praeco-helm) - -[Installing Praeco (ElastAlert GUI) into Kubernetes with Helm](https://en-designetwork.daichi703n.com/entry/2020/02/24/praeco-helm-kubernetes) - -### Praeco -![Praeco screenshot](https://user-images.githubusercontent.com/611996/47752071-7c4a9080-dc61-11e8-8ccf-2196f13429b2.png) -[Praeco](https://github.com/johnsusek/praeco) is a free open source GUI for ElastAlert. +The original [yelp/elastalert][0] repository has become mostly stale, with hundreds of open PRs and +over 1000 open issues. The Yelp team has acknowledged that they are winding down support of +Elastalert. Consequently, it is difficult to merge fixes, dependency upgrades, and new features into +Elastalert. Because of this, a fork of Elastalert has been created. [jertel/elastalert][1] will be +an alternate repository for updates, until a new maintainer is appointed by the Yelp team and it's +clear that the new maintainers are responding to PRs and issues. ## Documentation -Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). - -To build a html version of the docs locally - -``` -pip install sphinx_rtd_theme sphinx -cd docs -make html -``` - -View in browser at build/html/index.html - -## Configuration - -See config.yaml.example for details on configuration. - -## Example rules - -Examples of different types of rules can be found in example_rules/. +Updated Elastalert documentation that reflects the state of the _alt_ branch can be found [here][3]. +This is the place to start if you're not familiar with Elastalert at all. -- ``example_spike.yaml`` is an example of the "spike" rule type, which allows you to alert when the rate of events, averaged over a time period, -increases by a given factor. This example will send an email alert when there are 3 times more events matching a filter occurring within the -last 2 hours than the number of events in the previous 2 hours. +The full list of platforms that Elastalert can fire alerts into can be found [here][4] -- ``example_frequency.yaml`` is an example of the "frequency" rule type, which will alert when there are a given number of events occuring -within a time period. This example will send an email when 50 documents matching a given filter occur within a 4 hour timeframe. +The original README for Elastalert can be found [here][5]. Please note that this file is +not being actively maintained, and will probably grow less accurate over time. -- ``example_change.yaml`` is an example of the "change" rule type, which will alert when a certain field in two documents changes. In this example, -the alert email is sent when two documents with the same 'username' field but a different value of the 'country_name' field occur within 24 hours -of each other. +## Contributing -- ``example_new_term.yaml`` is an example of the "new term" rule type, which alerts when a new value appears in a field or fields. In this example, -an email is sent when a new value of ("username", "computer") is encountered in example login logs. +PRs are welcome, but must include tests, when possible. PRs will not be merged if they do not pass +the automated CI workflows. -## Frequently Asked Questions - -### My rule is not getting any hits? - -So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like - -``` -filter: -- query: - query_string: - query: "foo: bar AND baz: abc*" -``` -If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example, - -``` -filter: -- term: - foo: "Test Document" -``` - -will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening. - -### I got hits, why didn't I get an alert? - -If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types. - -If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs. - -If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" as the "From:" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this. - -### Why did I only get one alert when I expected to get several? - -There is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use - -``` -realert: - minutes: 0 -``` - -You can of course set it higher as well. - -### How can I prevent duplicate alerts? - -By setting ``realert``, you will prevent the same rule from alerting twice in an amount of time. - -``` -realert: - days: 1 -``` - -You can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use - -``` -realert: - hours: 8 -query_key: user -``` - -Note that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use - -``` -query_key: [error, hostname] -``` - -Internally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``. - -The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory. - -### How can I change what's in the alert? - -You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: - -With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. - -``` -alert_text_type: alert_text_jinja - -alert_text: | - Alert triggered! *({{num_hits}} Matches!)* - Something happened with {{username}} ({{email}}) - {{description|truncate}} - -``` - -> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. -> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. - -With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example - -``` -alert_text: "Something happened with {0} at {1}" -alert_text_type: alert_text_only -alert_text_args: ["username", "@timestamp"] -``` - -You can also limit the alert to only containing certain fields from the document by using ``include``. - -``` -include: ["ip_address", "hostname", "status"] -``` - -### My alert only contains data for one event, how can I see more? - -If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have - -``` -top_count_keys: ["ip_address", "status"] -``` - -and 10 documents matched your alert, it may contain something like - -``` -ip_address: -127.0.0.1: 7 -10.0.0.1: 2 -192.168.0.1: 1 - -status: -200: 9 -500: 1 -``` - -### How can I make the alert come at a certain time? - -The ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using - -``` -aggregation: - schedule: '2 4 * * mon,fri' -``` - -### I have lots of documents and it's really slow, how can I speed it up? - -There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` - -``` -index: logstash-%Y.%m -use_strftime_index: true -``` - -Another thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes. - -``` -buffer_time: - minutes: 5 -``` - -By default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp. - -If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types. - -### Can I perform aggregations? - -The only aggregation supported currently is a terms aggregation, by setting ``use_terms_query``. - -### I'm not using @timestamp, what do I do? - -You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents. - -### I'm using flatline but I don't see any alerts - -When using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them. - -### How can I get a "resolve" event? +The current status of the _alt_ branch CI workflow: -ElastAlert does not currently support stateful alerts or resolve events. +![CI Workflow](https://github.com/jertel/elastalert/workflows/alt_build_test/badge.svg) -### Can I set a warning threshold? +## Docker -Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. +If you're interested in a pre-built Docker image for either the official yelp/elastalert release, or +for this fork, check out the [elastalert-docker][2] project on Docker Hub. ## License -ElastAlert is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - -### Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). +Elastalert is licensed under the [Apache License, Version 2.0][6]. -### Questions? Drop by #elastalert on Freenode IRC. +[0]: https://github.com/yelp/elastalert +[1]: https://github.com/jertel/elastalert +[2]: https://hub.docker.com/r/jertel/elastalert-docker +[3]: https://elastalert2.readthedocs.io/ +[4]: https://elastalert2.readthedocs.io/en/latest/ruletypes.html#alerts +[5]: https://github.com/jertel/elastalert/blob/alt/README-old.md +[6]: http://www.apache.org/licenses/LICENSE-2 From 4f947691cbcdada0e3402d8f8f48426ac836ca92 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 2 Mar 2021 08:36:07 +0900 Subject: [PATCH 085/128] Bump coverage from 5.4 to 5.5 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b0642e5..d81c7ca2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt -coverage==5.4 +coverage==5.5 flake8 pluggy>=0.12.0 pre-commit From 2f36a0866d2a646a75710ef10c0310e72586c049 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Tue, 2 Mar 2021 08:03:09 +0000 Subject: [PATCH 086/128] Remove mention of new alert platforms from the old README As requested by the author of these alerters, @nsano-rururu --- README-old.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README-old.md b/README-old.md index bbdf39d9..9902b4d7 100644 --- a/README-old.md +++ b/README-old.md @@ -67,9 +67,6 @@ Currently, we have built-in support for the following alert types: - Line Notify - TheHive - Zabbix -- Discord -- Dingtalk -- Chatwork Additional rule types and alerts can be easily imported or written. From d3fd9bb3e9f4624822e73f1e7306c648ec4da2c8 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Tue, 2 Mar 2021 09:19:13 +0000 Subject: [PATCH 087/128] Migrate the FAQ to Read the Docs - Also add a short README on building the documentation - Make some tweaks to allow the generation of documentation from Markdown rather than ReStructured Text --- docs/README.md | 12 +++ docs/source/conf.py | 4 +- docs/source/index.rst | 1 + docs/source/recipes/faq.md | 181 ++++++++++++++++++++++++++++++++++++ docs/source/recipes/faq.rst | 6 ++ 5 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/source/recipes/faq.md create mode 100644 docs/source/recipes/faq.rst diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..eab45524 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# Documentation + +You can read this documentation at [Read The Docs][0]. + +To build a local version of these docs, the following from within the `/docs` directory: + +``` +pip install m2r2 sphinx_rtd_theme sphinx +make html +``` + +You can then view the generated HTML in from within the `build/` folder. diff --git a/docs/source/conf.py b/docs/source/conf.py index 80a76ed1..58de3fdd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,13 +19,13 @@ # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ["m2r2"] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The encoding of source files. # source_encoding = 'utf-8' diff --git a/docs/source/index.rst b/docs/source/index.rst index cf6d439c..17b15631 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,6 +22,7 @@ Contents: recipes/adding_enhancements recipes/adding_loaders recipes/signing_requests + recipes/faq Indices and Tables ================== diff --git a/docs/source/recipes/faq.md b/docs/source/recipes/faq.md new file mode 100644 index 00000000..8c2cbe63 --- /dev/null +++ b/docs/source/recipes/faq.md @@ -0,0 +1,181 @@ +My rule is not getting any hits? +========== + +So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like + +``` +filter: +- query: + query_string: + query: "foo: bar AND baz: abc*" +``` +If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example, + +``` +filter: +- term: + foo: "Test Document" +``` + +will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening. + +I got hits, why didn't I get an alert? +========== + +If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types. + +If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs. + +If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" as the "From:" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this. + +Why did I only get one alert when I expected to get several? +========== + +There is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use + +``` +realert: + minutes: 0 +``` + +You can of course set it higher as well. + +How can I prevent duplicate alerts? +========== + +By setting ``realert``, you will prevent the same rule from alerting twice in an amount of time. + +``` +realert: + days: 1 +``` + +You can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use + +``` +realert: + hours: 8 +query_key: user +``` + +Note that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use + +``` +query_key: [error, hostname] +``` + +Internally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``. + +The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory. + +How can I change what's in the alert? +========== + +You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: + +With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. + +``` +alert_text_type: alert_text_jinja + +alert_text: | + Alert triggered! *({{num_hits}} Matches!)* + Something happened with {{username}} ({{email}}) + {{description|truncate}} + +``` + +> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. +> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. + +With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example + +``` +alert_text: "Something happened with {0} at {1}" +alert_text_type: alert_text_only +alert_text_args: ["username", "@timestamp"] +``` + +You can also limit the alert to only containing certain fields from the document by using ``include``. + +``` +include: ["ip_address", "hostname", "status"] +``` + +My alert only contains data for one event, how can I see more? +========== + +If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have + +``` +top_count_keys: ["ip_address", "status"] +``` + +and 10 documents matched your alert, it may contain something like + +``` +ip_address: +127.0.0.1: 7 +10.0.0.1: 2 +192.168.0.1: 1 + +status: +200: 9 +500: 1 +``` + +How can I make the alert come at a certain time? +========== + +The ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using + +``` +aggregation: + schedule: '2 4 * * mon,fri' +``` + +I have lots of documents and it's really slow, how can I speed it up? +========== + +There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` + +``` +index: logstash-%Y.%m +use_strftime_index: true +``` + +Another thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes. + +``` +buffer_time: + minutes: 5 +``` + +By default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp. + +If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types. + +Can I perform aggregations? +========== + +The only aggregation supported currently is a terms aggregation, by setting ``use_terms_query``. + +I'm not using @timestamp, what do I do? +========== + +You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents. + +I'm using flatline but I don't see any alerts +========== + +When using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them. + +How can I get a "resolve" event? +========== + +ElastAlert does not currently support stateful alerts or resolve events. + +Can I set a warning threshold? +========== + +Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. diff --git a/docs/source/recipes/faq.rst b/docs/source/recipes/faq.rst new file mode 100644 index 00000000..ecbd5ad0 --- /dev/null +++ b/docs/source/recipes/faq.rst @@ -0,0 +1,6 @@ +.. _faq: + +Frequently Asked Questions +-------------------------- + +.. mdinclude:: faq.md From d701105e512597b1225426f03c0df7cb868e1362 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 2 Mar 2021 19:12:56 +0900 Subject: [PATCH 088/128] Docker test python 3.6 to 3.9 --- Dockerfile-test | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index fb8a7840..2c04ca85 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,9 @@ FROM ubuntu:latest -RUN apt-get update && apt-get upgrade -y -RUN apt-get install software-properties-common -y +RUN apt update && apt upgrade -y +RUN apt install software-properties-common -y RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git +RUN apt -y install build-essential python3.9 python3.9-dev python3-pip libssl-dev git WORKDIR /home/elastalert diff --git a/tox.ini b/tox.ini index 71099e17..f5e3369b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py36,docs +envlist = py39,docs [testenv] deps = -rrequirements-dev.txt From 0e7d68cc472bf854b6105efb4fdfdfcbbfc65a51 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Tue, 2 Mar 2021 11:23:55 +0000 Subject: [PATCH 089/128] Add Markdown converter to dev dependency list --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index d81c7ca2..75ba1002 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r requirements.txt coverage==5.5 flake8 +m2r2 pluggy>=0.12.0 pre-commit pylint<2.8 From 951c6c1a540d57b2db5c9bdca447229fa1966d60 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Tue, 2 Mar 2021 11:28:08 +0000 Subject: [PATCH 090/128] Format line lengths for readability --- docs/source/recipes/faq.md | 118 ++++++++++++++++++++++++++++--------- 1 file changed, 91 insertions(+), 27 deletions(-) diff --git a/docs/source/recipes/faq.md b/docs/source/recipes/faq.md index 8c2cbe63..86ba461a 100644 --- a/docs/source/recipes/faq.md +++ b/docs/source/recipes/faq.md @@ -1,7 +1,13 @@ My rule is not getting any hits? ========== -So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like +So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says +``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to +debug. It will show you how many documents match your filters for the last 24 hours (or more, see +``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, +remove it and try again. This will show you if the index is correct and that you have at least some +documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want +to use a query string. Your filter will look like ``` filter: @@ -9,7 +15,10 @@ filter: query_string: query: "foo: bar AND baz: abc*" ``` -If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example, +If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced +correctly, and the filter is not in the right format. If you are using other types of filters, like +``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the +default if you are using Logstash. For example, ``` filter: @@ -17,21 +26,41 @@ filter: foo: "Test Document" ``` -will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening. +will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want +to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, +try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made +to Elasticsearch in full so that you can see exactly what is happening. I got hits, why didn't I get an alert? ========== -If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types. - -If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs. - -If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" as the "From:" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this. +If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why +you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using +``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to +occur. Different rules apply for different rule types. + +If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set +``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an +alert for this same rule before, that rule may be silenced for a period of time. The default is one +minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in +the logs. + +If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert +configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the +alert text will be written to the console. Use ``--verbose`` to achieve the same affects without +preventing alerts. If you are using email alert, make sure you have it configured for an SMTP +server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" +as the "From:" address. Some SMTP servers will reject this because it does not have a domain while +others will add their own domain automatically. See the email section in the documentation for how +to configure this. Why did I only get one alert when I expected to get several? ========== -There is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use +There is a setting called ``realert`` which is the minimum time between two alerts for the same +rule. Any alert that occurs within this time will simply be dropped. The default value for this is +one minute. If you want to receive an alert for every single match, even if they occur right after +each other, use ``` realert: @@ -50,7 +79,8 @@ realert: days: 1 ``` -You can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use +You can also prevent duplicates based on a certain field by using ``query_key``. For example, to +prevent multiple alerts for the same user, you might use ``` realert: @@ -58,22 +88,30 @@ realert: query_key: user ``` -Note that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use +Note that this will also affect the way many rule types work. If you are using ``type: frequency`` +for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be +sent. You can also use a compound of multiple fields for this key. For example, if you only wanted +to receieve an alert once for a specific error and hostname, you could use ``` query_key: [error, hostname] ``` -Internally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``. +Internally, this works by creating a new field for each document called ``field1,field2`` with a +value of ``value1,value2`` and using that as the ``query_key``. -The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory. +The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` +index, with a ``_type`` of ``silence`` and also cached in memory. How can I change what's in the alert? ========== -You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: +You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: +alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. +You can also add different fields from the alert: -With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. +With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) +Template. ``` alert_text_type: alert_text_jinja @@ -85,10 +123,12 @@ alert_text: | ``` -> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. +> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful + when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. > If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. -With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example +With ``alert_text_type: alert_text_only`` by using Python style string formatting and +``alert_text_args``. For example ``` alert_text: "Something happened with {0} at {1}" @@ -96,7 +136,8 @@ alert_text_type: alert_text_only alert_text_args: ["username", "@timestamp"] ``` -You can also limit the alert to only containing certain fields from the document by using ``include``. +You can also limit the alert to only containing certain fields from the document by using +``include``. ``` include: ["ip_address", "hostname", "status"] @@ -105,7 +146,10 @@ include: ["ip_address", "hostname", "status"] My alert only contains data for one event, how can I see more? ========== -If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have +If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every +document will be included in the alert. An alternative, which works for every type, is +``top_count_keys``. This will show the top counts for each value for certain fields. For example, if +you have ``` top_count_keys: ["ip_address", "status"] @@ -127,7 +171,9 @@ status: How can I make the alert come at a certain time? ========== -The ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using +The ``aggregation`` feature will take every alert that has occured over a period of time and send +them together in one alert. You can use cron style syntax to send all alerts that have occured since +the last once by using ``` aggregation: @@ -137,23 +183,36 @@ aggregation: I have lots of documents and it's really slow, how can I speed it up? ========== -There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` +There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, +Elasticsearch will query all shards, even if they do not possibly contain data with the correct +timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` ``` index: logstash-%Y.%m use_strftime_index: true ``` -Another thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes. +Another thing you could change is ``buffer_time``. By default, ElastAlert will query large +overlapping windows in order to ensure that it does not miss any events, even if they are indexed in +real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most +recent few minutes. ``` buffer_time: minutes: 5 ``` -By default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp. +By default, ElastAlert will download every document in full before processing them. Instead, you can +have ElastAlert simply get a count of the number of documents that have occured in between each +query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, +because ElastAlert will not know the contents of each documents, just the total number of them. This +also reduces the precision of alerts, because all events that occur between each query will be +rounded to a single timestamp. -If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types. +If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. +This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain +field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the +``_type`` of the documents. They may not be compatible with all rule types. Can I perform aggregations? ========== @@ -163,12 +222,16 @@ The only aggregation supported currently is a terms aggregation, by setting ``us I'm not using @timestamp, what do I do? ========== -You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents. +You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can +use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of +timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` +and ``buffer_time`` to adjust when ElastAlert will look for documents. I'm using flatline but I don't see any alerts ========== -When using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them. +When using ``type: flatline``, ElastAlert must see at least one document before it will alert you +that it has stopped seeing them. How can I get a "resolve" event? ========== @@ -178,4 +241,5 @@ ElastAlert does not currently support stateful alerts or resolve events. Can I set a warning threshold? ========== -Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. +Currently, the only way to set a warning threshold is by creating a second rule with a lower +threshold. From 31f04f4610c8426350416b34ac9cfda194930424 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Tue, 2 Mar 2021 15:28:45 +0000 Subject: [PATCH 091/128] Update the docs configuration to work better with Markdown The ReStructured Text parser doesn't like the Markdown file having the same name as the .md, so I've renamed the .md, and excluded it from being built. --- docs/source/conf.py | 1 + docs/source/recipes/{faq.md => faq-md.md} | 5 +++-- docs/source/recipes/faq.rst | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) rename docs/source/recipes/{faq.md => faq-md.md} (98%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 58de3fdd..4a7ac542 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -62,6 +62,7 @@ # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] +exclude_patterns = ['recipes/*.md'] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None diff --git a/docs/source/recipes/faq.md b/docs/source/recipes/faq-md.md similarity index 98% rename from docs/source/recipes/faq.md rename to docs/source/recipes/faq-md.md index 86ba461a..15f87fba 100644 --- a/docs/source/recipes/faq.md +++ b/docs/source/recipes/faq-md.md @@ -123,9 +123,9 @@ alert_text: | ``` -> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful +- Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. -> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. +- If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example @@ -243,3 +243,4 @@ Can I set a warning threshold? Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. + diff --git a/docs/source/recipes/faq.rst b/docs/source/recipes/faq.rst index ecbd5ad0..34a632f6 100644 --- a/docs/source/recipes/faq.rst +++ b/docs/source/recipes/faq.rst @@ -3,4 +3,4 @@ Frequently Asked Questions -------------------------- -.. mdinclude:: faq.md +.. mdinclude:: faq-md.md From fb457532b974343b93a2dfeb75cd23d1fb83c1db Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 4 Mar 2021 21:54:16 +0900 Subject: [PATCH 092/128] Bump tox from 3.22.0 to 3.23.0 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 75ba1002..5ceb8762 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,4 +8,4 @@ pylint<2.8 pytest<3.7.0 setuptools sphinx_rtd_theme -tox==3.22.0 +tox==3.23.0 From f95b9d8e4e30ba7aa25c43295e83e346f223b37d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 14 Mar 2021 13:30:01 +0900 Subject: [PATCH 093/128] fix docs slack_timeout --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 1f99f22a..5eb1efac 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1808,7 +1808,7 @@ Provide absolute address of the pciture. ``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. -``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. +``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slack. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. ``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. From e438760cde0a0274dd69373112f85020679195f9 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 14 Mar 2021 14:06:30 +0900 Subject: [PATCH 094/128] Remove unused items in alerta from schema.yaml alerta_new_style_string_format alerta_customer --- elastalert/schema.yaml | 4 +--- tests/alerts_test.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 65f3dc3e..656be87e 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -352,14 +352,12 @@ properties: alerta_correlate: {type: array, items: {type: string}} # Python format string alerta_tags: {type: array, items: {type: string}} # Python format string alerta_event: {type: string} # Python format string - alerta_customer: {type: string} alerta_text: {type: string} # Python format string alerta_type: {type: string} alerta_value: {type: string} # Python format string alerta_attributes_keys: {type: array, items: {type: string}} alerta_attributes_values: {type: array, items: {type: string}} # Python format string - alerta_new_style_string_format: {type: boolean} - + ### Simple simple_webhook_url: *arrayOfString diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 11190fdd..c0221ed1 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -2220,7 +2220,6 @@ def test_alerta_new_style(ea): 'alerta_severity': "debug", 'alerta_text': "Probe {hostname} is UP at {logdate} GMT", 'alerta_value': "UP", - 'alerta_new_style_string_format': True, 'type': 'any', 'alerta_use_match_timestamp': True, 'alert': 'alerta' From a33afd39d86203f5af04e50a9b9db5098dac4182 Mon Sep 17 00:00:00 2001 From: Naoyuki Sano Date: Sun, 14 Mar 2021 14:11:29 +0900 Subject: [PATCH 095/128] Update schema.yaml --- elastalert/schema.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 656be87e..5e95ab86 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -357,7 +357,6 @@ properties: alerta_value: {type: string} # Python format string alerta_attributes_keys: {type: array, items: {type: string}} alerta_attributes_values: {type: array, items: {type: string}} # Python format string - ### Simple simple_webhook_url: *arrayOfString From cde1644e3724cd2a805406cb719e7db013ac1d85 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Mon, 15 Mar 2021 08:24:01 +0000 Subject: [PATCH 096/128] Fix search syntax on aggregations This brings in a change that's been applied on other forks (and opened on the main Elastalert repo but not merged): https://github.com/skillz/elastalert/pull/1 https://github.com/Yelp/elastalert/pull/2038 Without this fix, syntax errors are generated when document IDs that contain '-' characters are seen by alerts that use the `aggregation` field. --- elastalert/elastalert.py | 2 +- tests/base_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index e689619f..57bd771c 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1748,7 +1748,7 @@ def get_aggregated_matches(self, _id): """ Removes and returns all matches from writeback_es that have aggregate_id == _id """ # XXX if there are more than self.max_aggregation matches, you have big alerts and we will leave entries in ES. - query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} + query = {'query': {'query_string': {'query': 'aggregate_id:"%s"' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: if self.writeback_es.is_atleastsixtwo(): diff --git a/tests/base_test.py b/tests/base_test.py index 92dc35f7..81724e72 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -427,8 +427,8 @@ def test_agg_matchtime(ea): call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] - assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' - assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' + assert call3['query']['query_string']['query'] == 'aggregate_id:"ABCD"' + assert call4['query']['query_string']['query'] == 'aggregate_id:"CDEF"' assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 From 172bd5f81713bed2f508cb931f56bacf41954b65 Mon Sep 17 00:00:00 2001 From: Feroz Salam Date: Mon, 15 Mar 2021 08:35:11 +0000 Subject: [PATCH 097/128] Update unit tests to match new query structure --- tests/base_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index 81724e72..b86498b1 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -596,8 +596,8 @@ def test_agg_with_aggregation_key(ea): call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] - assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' - assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' + assert call3['query']['query_string']['query'] == 'aggregate_id:"ABCD"' + assert call4['query']['query_string']['query'] == 'aggregate_id:"CDEF"' assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 From 14cada534f16dca84783c4e3f4cf70f49a9426ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Mar 2021 05:39:52 +0000 Subject: [PATCH 098/128] Bump jinja2 from 2.10.1 to 2.11.3 Bumps [jinja2](https://github.com/pallets/jinja) from 2.10.1 to 2.11.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.1...2.11.3) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 52a32fd5..99e3306e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ croniter>=0.3.16 elasticsearch==7.0.0 envparse>=0.2.0 exotel>=0.1.3 -Jinja2==2.10.1 +Jinja2==2.11.3 jira>=2.0.0 jsonschema>=3.0.2 mock>=2.0.0 diff --git a/setup.py b/setup.py index 4754843a..e0e320ee 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=2.0.0', - 'Jinja2==2.10.1', + 'Jinja2==2.11.3', 'jsonschema>=3.0.2', 'mock>=2.0.0', 'prison>=0.1.2', From 0cfe1bf1bafa5d74e59a9620031048f96c1fe120 Mon Sep 17 00:00:00 2001 From: Alice Date: Sun, 21 Mar 2021 19:36:48 +0000 Subject: [PATCH 099/128] allow loading jinja templates from filesystem - and to extend templates --- elastalert/loaders.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index bee5d475..cecdd8ad 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -9,6 +9,8 @@ import yaml import yaml.scanner from jinja2 import Template +from jinja2 import Environment +from jinja2 import FileSystemLoader from staticconf.loader import yaml_loader from . import alerts @@ -95,6 +97,8 @@ class RulesLoader(object): base_config = {} + jinja_environment = Environment(loader=FileSystemLoader("")) + def __init__(self, conf): # schema for rule yaml self.rule_schema = jsonschema.Draft7Validator( @@ -401,7 +405,11 @@ def _dt_to_ts_with_format(dt): # Compile Jinja Template if rule.get('alert_text_type') == 'alert_text_jinja': - rule["jinja_template"] = Template(str(rule.get('alert_text', ''))) + jinja_template_path = rule.get('jinja_template_path') + if jinja_template_path: + rule["jinja_template"] = self.jinja_environment.get_or_select_template(jinja_template_path) + else: + rule["jinja_template"] = Template(str(rule.get('alert_text', ''))) def load_modules(self, rule, args=None): """ Loads things that could be modules. Enhancements, alerts and rule type. """ From f15b8aa45680503c7537de44259ba01aa64987c5 Mon Sep 17 00:00:00 2001 From: Alice Date: Sun, 21 Mar 2021 20:30:49 +0000 Subject: [PATCH 100/128] update docs --- docs/source/elastalert.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 17dbf5b0..6b5bfe31 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -220,6 +220,8 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``jinja_root_name``: When using a Jinja template, specify the name of the root field name in the template. The default is ``_data``. +``jinja_template_path``: When using a Jinja template, specify filesystem path to template, this overrides the default behaviour of using alert_text as the template. + Logging ------- From 24c6c982241a9ebdf2a8ce12cbae5f34bae6f1e4 Mon Sep 17 00:00:00 2001 From: Evan Lock Date: Fri, 2 Apr 2021 09:40:10 -0400 Subject: [PATCH 101/128] Add alert handler to create Datadog Events --- elastalert/alerts.py | 30 ++++++++++++++++++++++++++++++ elastalert/loaders.py | 3 ++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f2f31853..17f2e730 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2184,3 +2184,33 @@ def get_info(self): 'type': 'hivealerter', 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') } + +class DatadogAlerter(Alerter): + ''' Creates a Datadog Event for each alert ''' + required_options = frozenset(['datadog_api_key', 'datadog_app_key']) + + def __init__(self, rule): + super(DatadogAlerter, self).__init__(rule) + self.dd_api_key = self.rule.get('datadog_api_key', None) + self.dd_app_key = self.rule.get('datadog_app_key', None) + + def alert(self, matches): + url = 'https://api.datadoghq.com/api/v1/events' + headers = { + 'Content-Type': 'application/json', + 'DD-API-KEY': self.dd_api_key, + 'DD-APPLICATION-KEY': self.dd_app_key + } + payload = { + 'title': self.create_title(matches), + 'text': self.create_alert_body(matches) + } + try: + response = requests.post(url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException('Error posting event to Datadog: %s' % e) + elastalert_logger.info('Alert sent to Datadog') + + def get_info(self): + return {'type': 'datadog'} diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 77119476..1aabd003 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,7 +77,8 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, - 'hivealerter': alerts.HiveAlerter + 'hivealerter': alerts.HiveAlerter, + 'datadog': alerts.DatadogAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list From 457501e2b0ef983cd0c613c29355f08019169647 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 4 Apr 2021 23:59:39 +0900 Subject: [PATCH 102/128] Kibana Discover support kibana 7.12 --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 5eb1efac..253a9dbc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -555,9 +555,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11`, `7.12` -``kibana_discover_version: '7.11'`` +``kibana_discover_version: '7.12'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 0cbbc211..58e3476f 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -15,7 +15,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 5e95ab86..771fc99a 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -219,7 +219,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.12', '7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index 30191b89..0e796e48 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From ab397683c9b49fe98b43669b7a3b86e5984ceba8 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 12 Apr 2021 23:44:40 +0900 Subject: [PATCH 103/128] apscheduler>=3.3.0_to_>=3.3.0,<4.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 99e3306e..a865b3f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apscheduler>=3.3.0 +apscheduler>=3.3.0,<4.0 aws-requests-auth>=0.3.0 sortedcontainers>=2.2.2 boto3>=1.4.4 diff --git a/setup.py b/setup.py index e0e320ee..85a102c6 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']}, install_requires=[ - 'apscheduler>=3.3.0', + 'apscheduler>=3.3.0,<4.0', 'aws-requests-auth>=0.3.0', 'sortedcontainers>=2.2.2', 'boto3>=1.4.4', From 45237cc364e2cafc1859d12841b1bab11c810d9f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 13 Apr 2021 23:45:08 +0900 Subject: [PATCH 104/128] Update sphinx and elasticsearch-py sphinx 1.6.6 to 3.5.4 elasticsearch 7.0.0 to >=7.0.0,<8.0.0 --- requirements.txt | 2 +- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index a865b3f4..67558b3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ boto3>=1.4.4 cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 -elasticsearch==7.0.0 +elasticsearch>=7.0.0,<8.0.0 envparse>=0.2.0 exotel>=0.1.3 Jinja2==2.11.3 diff --git a/setup.py b/setup.py index 85a102c6..cd264498 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', - 'elasticsearch==7.0.0', + 'elasticsearch>=7.0.0,<8.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=2.0.0', diff --git a/tox.ini b/tox.ini index f5e3369b..47e62caa 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,6 @@ norecursedirs = .* virtualenv_run docs build venv env [testenv:docs] deps = {[testenv]deps} - sphinx==1.6.6 + sphinx==3.5.4 changedir = docs commands = sphinx-build -b html -d build/doctrees -W source build/html From 955f75742dd532069bbe230564e9fdc0757b0e57 Mon Sep 17 00:00:00 2001 From: "GIBSON, NICHOLAS R" Date: Fri, 16 Apr 2021 16:26:41 -0700 Subject: [PATCH 105/128] added optional Prometheus metrics endpoint --- README-old.md | 2 ++ elastalert/elastalert.py | 8 +++++ elastalert/prometheus_wrapper.py | 55 ++++++++++++++++++++++++++++++++ requirements.txt | 1 + setup.py | 1 + 5 files changed, 67 insertions(+) create mode 100644 elastalert/prometheus_wrapper.py diff --git a/README-old.md b/README-old.md index 9902b4d7..982ebbc4 100644 --- a/README-old.md +++ b/README-old.md @@ -115,6 +115,8 @@ Eg: ``--rule this_rule.yaml`` ``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory. +``--prometheus_port`` exposes ElastAlert Prometheus metrics on the specified port. Prometheus metrics disabled by default. + ## Third Party Tools And Extras ### Kibana plugin ![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 57bd771c..c7509a7c 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -25,6 +25,7 @@ from elasticsearch.exceptions import ElasticsearchException from elasticsearch.exceptions import NotFoundError from elasticsearch.exceptions import TransportError +from .prometheus_wrapper import PrometheusWrapper from . import kibana from .alerts import DebugAlerter @@ -111,6 +112,7 @@ def parse_args(self, args): dest='es_debug_trace', help='Enable logging from Elasticsearch queries as curl command. Queries will be logged to file. Note that ' 'this will incorrectly display localhost:9200 as the host/port') + parser.add_argument('--prometheus_port', type=int, dest='prometheus_port', help='Enables Prometheus metrics on specified port.') self.args = parser.parse_args(args) def __init__(self, args): @@ -171,6 +173,7 @@ def __init__(self, args): self.scheduler = BackgroundScheduler() self.string_multi_field_name = self.conf.get('string_multi_field_name', False) self.add_metadata_alert = self.conf.get('add_metadata_alert', False) + self.prometheus_port = self.args.prometheus_port self.show_disabled_rules = self.conf.get('show_disabled_rules', True) self.writeback_es = elasticsearch_client(self.conf) @@ -2076,6 +2079,11 @@ def main(args=None): if not args: args = sys.argv[1:] client = ElastAlerter(args) + + if client.prometheus_port and not client.debug: + p = PrometheusWrapper(client) + p.start() + if not client.args.silence: client.start() diff --git a/elastalert/prometheus_wrapper.py b/elastalert/prometheus_wrapper.py new file mode 100644 index 00000000..b3d0758b --- /dev/null +++ b/elastalert/prometheus_wrapper.py @@ -0,0 +1,55 @@ +import prometheus_client + + +class PrometheusWrapper: + """ Exposes ElastAlert metrics on a Prometheus metrics endpoint. + Wraps ElastAlerter run_rule and writeback to collect metrics. """ + + def __init__(self, client): + self.prometheus_port = client.prometheus_port + self.run_rule = client.run_rule + self.writeback = client.writeback + + client.run_rule = self.metrics_run_rule + client.writeback = self.metrics_writeback + + # initialize prometheus metrics to be exposed + self.prom_scrapes = prometheus_client.Counter('elastalert_scrapes', 'Number of scrapes for rule', ['rule_name']) + self.prom_hits = prometheus_client.Counter('elastalert_hits', 'Number of hits for rule', ['rule_name']) + self.prom_matches = prometheus_client.Counter('elastalert_matches', 'Number of matches for rule', ['rule_name']) + self.prom_time_taken = prometheus_client.Counter('elastalert_time_taken', 'Time taken to evaluate rule', ['rule_name']) + self.prom_alerts_sent = prometheus_client.Counter('elastalert_alerts_sent', 'Number of alerts sent for rule', ['rule_name']) + self.prom_alerts_not_sent = prometheus_client.Counter('elastalert_alerts_not_sent', 'Number of alerts not sent', ['rule_name']) + self.prom_errors = prometheus_client.Counter('elastalert_errors', 'Number of errors for rule') + self.prom_alerts_silenced = prometheus_client.Counter('elastalert_alerts_silenced', 'Number of silenced alerts', ['rule_name']) + + def start(self): + prometheus_client.start_http_server(self.prometheus_port) + + def metrics_run_rule(self, rule, endtime, starttime=None): + """ Increment counter every time rule is run """ + try: + self.prom_scrapes.labels(rule['name']).inc() + finally: + return self.run_rule(rule, endtime, starttime) + + def metrics_writeback(self, doc_type, body): + """ Update various prometheus metrics accoording to the doc_type """ + + res = self.writeback(doc_type, body) + try: + if doc_type == 'elastalert_status': + self.prom_hits.labels(body['rule_name']).inc(int(body['hits'])) + self.prom_matches.labels(body['rule_name']).inc(int(body['matches'])) + self.prom_time_taken.labels(body['rule_name']).inc(float(body['time_taken'])) + elif doc_type == 'elastalert': + if body['alert_sent']: + self.prom_alerts_sent.labels(body['rule_name']).inc() + else: + self.prom_alerts_not_sent.labels(body['rule_name']).inc() + elif doc_type == 'elastalert_error': + self.prom_errors.inc() + elif doc_type == 'silence': + self.prom_alerts_silenced.labels(body['rule_name']).inc() + finally: + return res \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 67558b3d..c35de8e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ jira>=2.0.0 jsonschema>=3.0.2 mock>=2.0.0 prison>=0.1.2 +prometheus_client>=0.10.1 py-zabbix>=1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 diff --git a/setup.py b/setup.py index cd264498..5abdd436 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'jsonschema>=3.0.2', 'mock>=2.0.0', 'prison>=0.1.2', + 'prometheus_client>=0.10.1', 'py-zabbix>=1.1.3', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', From 606f89e2cb8c8ca0685e803bbbcf33aebbb99f3f Mon Sep 17 00:00:00 2001 From: "GIBSON, NICHOLAS R" Date: Mon, 19 Apr 2021 10:14:49 -0700 Subject: [PATCH 106/128] fix build issues --- elastalert/elastalert.py | 2 +- elastalert/prometheus_wrapper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index c7509a7c..dc910c6b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -2079,7 +2079,7 @@ def main(args=None): if not args: args = sys.argv[1:] client = ElastAlerter(args) - + if client.prometheus_port and not client.debug: p = PrometheusWrapper(client) p.start() diff --git a/elastalert/prometheus_wrapper.py b/elastalert/prometheus_wrapper.py index b3d0758b..d94a3520 100644 --- a/elastalert/prometheus_wrapper.py +++ b/elastalert/prometheus_wrapper.py @@ -52,4 +52,4 @@ def metrics_writeback(self, doc_type, body): elif doc_type == 'silence': self.prom_alerts_silenced.labels(body['rule_name']).inc() finally: - return res \ No newline at end of file + return res From 63a7cc41d3a42f939556244a48608062aa20ddbe Mon Sep 17 00:00:00 2001 From: just1900 Date: Wed, 31 Mar 2021 10:42:45 +0800 Subject: [PATCH 107/128] fix compound query key in metric aggregation --- elastalert/ruletypes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index c0591275..57560754 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1122,7 +1122,13 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou result, compound_keys[1:], match_data) - + elif 'interval_aggs' in aggregation_data: + for result in aggregation_data['interval_aggs']['buckets']: + self.check_matches_recursive(timestamp, + query_key, + result, + compound_keys[1:], + match_data) else: metric_val = aggregation_data[self.metric_key]['value'] if self.crossed_thresholds(metric_val): From 3d0a285a509e35589cabfe751be9b6f30565d106 Mon Sep 17 00:00:00 2001 From: just1900 Date: Thu, 22 Apr 2021 23:07:14 +0800 Subject: [PATCH 108/128] Unit Test added for metric_aggregation with complex query key and bucket interval --- elastalert/ruletypes.py | 30 +++++++++++++----------------- tests/rules_test.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 57560754..39719739 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1122,24 +1122,20 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou result, compound_keys[1:], match_data) - elif 'interval_aggs' in aggregation_data: - for result in aggregation_data['interval_aggs']['buckets']: - self.check_matches_recursive(timestamp, - query_key, - result, - compound_keys[1:], - match_data) else: - metric_val = aggregation_data[self.metric_key]['value'] - if self.crossed_thresholds(metric_val): - match_data[self.rules['timestamp_field']] = timestamp - match_data[self.metric_key] = metric_val - - # add compound key to payload to allow alerts to trigger for every unique occurence - compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) - - self.add_match(match_data) + if 'interval_aggs' in aggregation_data: + metric_val_arr = [term[self.metric_key]['value'] for term in aggregation_data['interval_aggs']['buckets']] + else: + metric_val_arr = [aggregation_data[self.metric_key]['value']] + for metric_val in metric_val_arr: + if self.crossed_thresholds(metric_val): + match_data[self.rules['timestamp_field']] = timestamp + match_data[self.metric_key] = metric_val + + # add compound key to payload to allow alerts to trigger for every unique occurence + compound_value = [match_data[key] for key in self.rules['compound_query_key']] + match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) + self.add_match(match_data) def crossed_thresholds(self, metric_value): if metric_value is None: diff --git a/tests/rules_test.py b/tests/rules_test.py index 1954b5d5..5f6421d8 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1184,6 +1184,41 @@ def test_metric_aggregation_complex_query_key(): assert rule.matches[1]['sub_qk'] == 'sub_qk_val2' +def test_metric_aggregation_complex_query_key_bucket_interval(): + rules = {'buffer_time': datetime.timedelta(minutes=5), + 'timestamp_field': '@timestamp', + 'metric_agg_type': 'avg', + 'metric_agg_key': 'cpu_pct', + 'bucket_interval': {'minutes': 1}, + 'bucket_interval_timedelta': datetime.timedelta(minutes=1), + 'compound_query_key': ['qk', 'sub_qk'], + 'query_key': 'qk,sub_qk', + 'max_threshold': 0.8} + + # Quoted from https://elastalert.readthedocs.io/en/latest/ruletypes.html#metric-aggregation + # bucket_interval: If present this will divide the metric calculation window into bucket_interval sized segments. + # The metric value will be calculated and evaluated against the threshold(s) for each segment. + interval_aggs = {"interval_aggs": {"buckets": [ + {"metric_cpu_pct_avg": {"value": 0.91}, "key": "1617156690000"}, + {"metric_cpu_pct_avg": {"value": 0.89}, "key": "1617156750000"}, + {"metric_cpu_pct_avg": {"value": 0.78}, "key": "1617156810000"}, + {"metric_cpu_pct_avg": {"value": 0.85}, "key": "1617156870000"}, + {"metric_cpu_pct_avg": {"value": 0.86}, "key": "1617156930000"}, + ]}, "key": "sub_qk_val1"} + + query = {"bucket_aggs": {"buckets": [ + interval_aggs + ]}, "key": "qk_val"} + + rule = MetricAggregationRule(rules) + rule.check_matches(datetime.datetime.now(), 'qk_val', query) + assert len(rule.matches) == 4 + assert rule.matches[0]['qk'] == 'qk_val' + assert rule.matches[1]['qk'] == 'qk_val' + assert rule.matches[0]['sub_qk'] == 'sub_qk_val1' + assert rule.matches[1]['sub_qk'] == 'sub_qk_val1' + + def test_percentage_match(): rules = {'match_bucket_filter': {'term': 'term_val'}, 'buffer_time': datetime.timedelta(minutes=5), From d348a2f55fe3170323e8800172c05726b85de1e8 Mon Sep 17 00:00:00 2001 From: Evan Lock Date: Fri, 23 Apr 2021 14:02:05 -0400 Subject: [PATCH 109/128] Add documentation for alert type --- docs/source/ruletypes.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 253a9dbc..62e5a4c6 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1489,6 +1489,18 @@ Example usage using new-style format:: command: ["/bin/send_alert", "--username", "{match[username]}"] +Datadog +~~~~~~~ + +This alert will create a [Datadog Event](https://docs.datadoghq.com/events/). Events are limited to 4000 characters. If an event is sent that contains +a message that is longer than 4000 characters, only his first 4000 characters will be displayed. + +This alert requires two additional options: + +``datadog_api_key``: [Datadog API key](https://docs.datadoghq.com/account_management/api-app-keys/#api-keys) + +``datadog_app_key``: [Datadog application key](https://docs.datadoghq.com/account_management/api-app-keys/#application-keys) + Email ~~~~~ From bb141fdc253214cb74aa77e11cc9ee58ec541802 Mon Sep 17 00:00:00 2001 From: Evan Lock Date: Fri, 23 Apr 2021 15:46:31 -0400 Subject: [PATCH 110/128] Add test for Datadog alert --- tests/alerts_test.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index c0221ed1..f66f0872 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -12,6 +12,7 @@ from elastalert.alerts import Alerter from elastalert.alerts import BasicMatchString from elastalert.alerts import CommandAlerter +from elastalert.alerts import DatadogAlerter from elastalert.alerts import EmailAlerter from elastalert.alerts import HTTPPostAlerter from elastalert.alerts import JiraAlerter @@ -2296,3 +2297,38 @@ def test_alert_subject_size_limit_with_args(ea): alert = Alerter(rule) alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) assert 6 == len(alertSubject) + +def test_datadog_alerter(): + rule = { + 'name': 'Test Datadog Event Alerter', + 'type': 'any', + 'datadog_api_key': 'test-api-key', + 'datadog_app_key': 'test-app-key', + 'alert': [], + 'alert_subject': 'Test Datadog Event Alert' + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = DatadogAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'name': 'datadog-test-name' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'title': rule['alert_subject'], + 'text': "Test Datadog Event Alerter\n\n@timestamp: 2021-01-01T00:00:00\nname: datadog-test-name\n" + } + mock_post_request.assert_called_once_with( + "https://api.datadoghq.com/api/v1/events", + data=mock.ANY, + headers={ + 'Content-Type': 'application/json', + 'DD-API-KEY': rule['datadog_api_key'], + 'DD-APPLICATION-KEY': rule['datadog_app_key'] + } + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data From a6e8673a1049d2223c2f86f96b3b225a958d9112 Mon Sep 17 00:00:00 2001 From: Evan Lock Date: Fri, 23 Apr 2021 15:59:06 -0400 Subject: [PATCH 111/128] Correct linting errors --- elastalert/alerts.py | 1 + tests/alerts_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 26d19008..db398844 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2232,6 +2232,7 @@ def get_info(self): "chatwork_room_id": self.chatwork_room_id } + class DatadogAlerter(Alerter): ''' Creates a Datadog Event for each alert ''' required_options = frozenset(['datadog_api_key', 'datadog_app_key']) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index f66f0872..4f56ad1a 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -2298,6 +2298,7 @@ def test_alert_subject_size_limit_with_args(ea): alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) assert 6 == len(alertSubject) + def test_datadog_alerter(): rule = { 'name': 'Test Datadog Event Alerter', From 12cba14c95ad64db8e69462c6754d16990c9db49 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 13:04:34 -0400 Subject: [PATCH 112/128] Update for Elastalert 2 --- README.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 694f03eb..2539d382 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,15 @@ -# Fork of yelp/elastalert +# Elastalert 2 -The original [yelp/elastalert][0] repository has become mostly stale, with hundreds of open PRs and -over 1000 open issues. The Yelp team has acknowledged that they are winding down support of -Elastalert. Consequently, it is difficult to merge fixes, dependency upgrades, and new features into -Elastalert. Because of this, a fork of Elastalert has been created. [jertel/elastalert][1] will be -an alternate repository for updates, until a new maintainer is appointed by the Yelp team and it's -clear that the new maintainers are responding to PRs and issues. +Elastalert 2 is the supported fork of [Elastalert][0], which had been maintained by the Yelp team +but become mostly stale when the Yelp team ceased using Elastalert. + +Elastalert 2 is backwards compatible with the original Elastalert rules. ## Documentation -Updated Elastalert documentation that reflects the state of the _alt_ branch can be found [here][3]. -This is the place to start if you're not familiar with Elastalert at all. +Documentation for Elastalert 2 can be found on [readthedocs.com][3]. This is the place to start if you're not familiar with Elastalert at all. -The full list of platforms that Elastalert can fire alerts into can be found [here][4] +The full list of platforms that Elastalert can fire alerts into can be found [in the documentation][4]. The original README for Elastalert can be found [here][5]. Please note that this file is not being actively maintained, and will probably grow less accurate over time. @@ -22,14 +19,14 @@ not being actively maintained, and will probably grow less accurate over time. PRs are welcome, but must include tests, when possible. PRs will not be merged if they do not pass the automated CI workflows. -The current status of the _alt_ branch CI workflow: +The current status of the CI workflow: -![CI Workflow](https://github.com/jertel/elastalert/workflows/alt_build_test/badge.svg) +![CI Workflow](https://github.com/jertel/elastalert/workflows/master_build_test/badge.svg) ## Docker If you're interested in a pre-built Docker image for either the official yelp/elastalert release, or -for this fork, check out the [elastalert-docker][2] project on Docker Hub. +for this fork, check out the [elastalert2][2] project on Docker Hub. ## License @@ -37,7 +34,7 @@ Elastalert is licensed under the [Apache License, Version 2.0][6]. [0]: https://github.com/yelp/elastalert [1]: https://github.com/jertel/elastalert -[2]: https://hub.docker.com/r/jertel/elastalert-docker +[2]: https://hub.docker.com/r/jertel/elastalert2 [3]: https://elastalert2.readthedocs.io/ [4]: https://elastalert2.readthedocs.io/en/latest/ruletypes.html#alerts [5]: https://github.com/jertel/elastalert/blob/alt/README-old.md From 9df402fc1b2658230354e18922b36d5b88792f24 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 13:07:44 -0400 Subject: [PATCH 113/128] Switch action to use master branch --- .../{alt_build_test.yml => master_build_test.yml} | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) rename .github/workflows/{alt_build_test.yml => master_build_test.yml} (86%) diff --git a/.github/workflows/alt_build_test.yml b/.github/workflows/master_build_test.yml similarity index 86% rename from .github/workflows/alt_build_test.yml rename to .github/workflows/master_build_test.yml index 7dab4f1c..8ab9d01a 100644 --- a/.github/workflows/alt_build_test.yml +++ b/.github/workflows/master_build_test.yml @@ -1,14 +1,12 @@ -# This is a basic workflow to help you get started with Actions - -name: alt_build_test +name: master_build_test # Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ alt ] + branches: [ master ] pull_request: - branches: [ alt ] + branches: [ master ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 7c1cff44893152ff0400ea61e3fdc317c5f609c2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:27:24 -0400 Subject: [PATCH 114/128] Merge elastalert-docker project; update README --- .github/workflows/release.yml | 19 + .secrets.baseline | 27 -- .travis.yml | 42 -- Dockerfile | 36 ++ README-old.md | 372 ----------------- README.md | 56 ++- changelog.md | 383 ------------------ chart/elastalert2/.helmignore | 21 + chart/elastalert2/Chart.yaml | 12 + chart/elastalert2/README.md | 90 ++++ chart/elastalert2/templates/NOTES.txt | 1 + chart/elastalert2/templates/_helpers.tpl | 36 ++ chart/elastalert2/templates/deployment.yaml | 124 ++++++ .../templates/podsecuritypolicy.yaml | 39 ++ chart/elastalert2/templates/role.yaml | 20 + chart/elastalert2/templates/rolebinding.yaml | 18 + chart/elastalert2/templates/rules.yaml | 14 + .../elastalert2/templates/serviceaccount.yaml | 15 + chart/elastalert2/values.yaml | 228 +++++++++++ setup.py | 10 +- 20 files changed, 724 insertions(+), 839 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100644 .secrets.baseline delete mode 100644 .travis.yml create mode 100644 Dockerfile delete mode 100644 README-old.md delete mode 100644 changelog.md create mode 100644 chart/elastalert2/.helmignore create mode 100644 chart/elastalert2/Chart.yaml create mode 100644 chart/elastalert2/README.md create mode 100644 chart/elastalert2/templates/NOTES.txt create mode 100644 chart/elastalert2/templates/_helpers.tpl create mode 100644 chart/elastalert2/templates/deployment.yaml create mode 100644 chart/elastalert2/templates/podsecuritypolicy.yaml create mode 100644 chart/elastalert2/templates/role.yaml create mode 100644 chart/elastalert2/templates/rolebinding.yaml create mode 100644 chart/elastalert2/templates/rules.yaml create mode 100644 chart/elastalert2/templates/serviceaccount.yaml create mode 100644 chart/elastalert2/values.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..5f51356b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,19 @@ +name: upload-chart + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Helm push chart + env: + HELM_REPO_ACCESS_TOKEN: ${{ secrets.HELM_REPO_ACCESS_TOKEN }} + run: | + wget https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz + tar xf helm-v3.2.1-linux-amd64.tar.gz + linux-amd64/helm plugin install https://github.com/chartmuseum/helm-push + linux-amd64/helm push chart/elastalert2 https://charts.banzaicloud.io/gh/Codesim-LLC diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index b4405a48..00000000 --- a/.secrets.baseline +++ /dev/null @@ -1,27 +0,0 @@ -{ - "exclude_regex": ".*tests/.*|.*yelp/testing/.*|\\.pre-commit-config\\.yaml", - "generated_at": "2018-07-06T22:54:22Z", - "plugins_used": [ - { - "base64_limit": 4.5, - "name": "Base64HighEntropyString" - }, - { - "hex_limit": 3, - "name": "HexHighEntropyString" - }, - { - "name": "PrivateKeyDetector" - } - ], - "results": { - ".travis.yml": [ - { - "hashed_secret": "4f7a1ea04dafcbfee994ee1d08857b8aaedf8065", - "line_number": 14, - "type": "Base64 High Entropy String" - } - ] - }, - "version": "0.9.1" -} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 569bf12d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: python -python: -- '3.6' -env: -- TOXENV=docs -- TOXENV=py36 -install: -- pip install tox -- > - if [[ -n "${ES_VERSION}" ]] ; then - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz - mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1 - ./elasticsearch-${ES_VERSION}/bin/elasticsearch & - fi -script: -- > - if [[ -n "${ES_VERSION}" ]] ; then - wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 - make test-elasticsearch - else - make test - fi -jobs: - include: - - stage: 'Elasticsearch test' - env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 - - env: TOXENV=py36 ES_VERSION=6.6.2 - - env: TOXENV=py36 ES_VERSION=6.3.2 - - env: TOXENV=py36 ES_VERSION=6.2.4 - - env: TOXENV=py36 ES_VERSION=6.0.1 - - env: TOXENV=py36 ES_VERSION=5.6.16 - -deploy: - provider: pypi - user: yelplabs - password: - secure: TpSTlFu89tciZzboIfitHhU5NhAB1L1/rI35eQTXstiqzYg2mweOuip+MPNx9AlX3Swg7MhaFYnSUvRqPljuoLjLD0EQ7BHLVSBFl92ukkAMTeKvM6LbB9HnGOwzmAvTR5coegk8IHiegudODWvnhIj4hp7/0EA+gVX7E55kEAw= - on: - tags: true - distributions: sdist bdist_wheel - repo: Yelp/elastalert - branch: master diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9b6924ce --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +FROM python:alpine as builder + +LABEL description="Elastalert 2 suitable for Kubernetes and Helm" +LABEL maintainer="Jason Ertel (jertel at codesim.com)" + +RUN apk --update upgrade && \ + apk add git && \ + rm -rf /var/cache/apk/* + +RUN mkdir -p /opt/elastalert && \ + git clone https://github.com/jertel/elastalert2 /tmp/elastalert && \ + cd /tmp/elastalert && \ + pip install setuptools wheel && \ + python setup.py sdist bdist_wheel + +FROM python:alpine + +COPY --from=builder /tmp/elastalert/dist/*.tar.gz /tmp/ + +RUN apk --update upgrade && \ + apk add gcc libffi-dev musl-dev python3-dev openssl-dev tzdata libmagic cargo && \ + pip install /tmp/*.tar.gz && \ + apk del gcc libffi-dev musl-dev python3-dev openssl-dev cargo && \ + rm -rf /var/cache/apk/* + +RUN mkdir -p /opt/elastalert && \ + echo "#!/bin/sh" >> /opt/elastalert/run.sh && \ + echo "set -e" >> /opt/elastalert/run.sh && \ + echo "elastalert-create-index --config /opt/config/elastalert_config.yaml" >> /opt/elastalert/run.sh && \ + echo "elastalert --config /opt/config/elastalert_config.yaml \"\$@\"" >> /opt/elastalert/run.sh && \ + chmod +x /opt/elastalert/run.sh + +ENV TZ "UTC" + +WORKDIR /opt/elastalert +ENTRYPOINT ["/opt/elastalert/run.sh"] diff --git a/README-old.md b/README-old.md deleted file mode 100644 index 982ebbc4..00000000 --- a/README-old.md +++ /dev/null @@ -1,372 +0,0 @@ -# Deprecated ElastAlert README - -*The documentation below refers to a previous ElastAlert version. For the latest version, see [here][0]* - -Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. - -[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) -[![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org). -### Easy & Flexible Alerting With Elasticsearch - -ElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. - -ElastAlert works with all versions of Elasticsearch. - -At Yelp, we use Elasticsearch, Logstash and Kibana for managing our ever increasing amount of data and logs. -Kibana is great for visualizing and querying data, but we quickly realized that it needed a companion tool for alerting -on inconsistencies in our data. Out of this need, ElastAlert was created. - -If you have data being written into Elasticsearch in near real time and want to be alerted when that data matches certain patterns, ElastAlert is the tool for you. If you can see it in Kibana, ElastAlert can alert on it. - -## Overview - -We designed ElastAlert to be reliable, highly modular, and easy to set up and configure. - -It works by combining Elasticsearch with two types of components, rule types and alerts. -Elasticsearch is periodically queried and the data is passed to the rule type, which determines when -a match is found. When a match occurs, it is given to one or more alerts, which take action based on the match. - -This is configured by a set of rules, each of which defines a query, a rule type, and a set of alerts. - -Several rule types with common monitoring paradigms are included with ElastAlert: - -- Match where there are at least X events in Y time" (``frequency`` type) -- Match when the rate of events increases or decreases" (``spike`` type) -- Match when there are less than X events in Y time" (``flatline`` type) -- Match when a certain field matches a blacklist/whitelist" (``blacklist`` and ``whitelist`` type) -- Match on any event matching a given filter" (``any`` type) -- Match when a field has two different values within some time" (``change`` type) -- Match when a never before seen term appears in a field" (``new_term`` type) -- Match when the number of unique values for a field is above or below a threshold (``cardinality`` type) - -Currently, we have built-in support for the following alert types: - -- Command -- Email -- JIRA -- OpsGenie -- AWS SNS -- MS Teams -- Slack -- Mattermost -- Telegram -- GoogleChat -- PagerDuty -- PagerTree -- Exotel -- Twilio -- Splunk On-Call (Formerly VictorOps) -- Gitter -- ServiceNow -- Debug -- Stomp -- Alerta -- HTTP POST -- Line Notify -- TheHive -- Zabbix - -Additional rule types and alerts can be easily imported or written. - -In addition to this basic usage, there are many other features that make alerts more useful: - -- Alerts link to Kibana dashboards -- Aggregate counts for arbitrary fields -- Combine alerts into periodic reports -- Separate alerts by using a unique key field -- Intercept and enhance match data - -To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). - -## Running ElastAlert -You can either install the latest released version of ElastAlert using pip: - -```pip install elastalert``` - -or you can clone the ElastAlert repository for the most recent changes: - -```git clone https://github.com/Yelp/elastalert.git``` - -Install the module: - -```pip install "setuptools>=11.3"``` - -```python setup.py install``` - -The following invocation can be used to run ElastAlert after installing - -``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` - -``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. - -``--verbose`` will print additional information without suppressing alerts. Not compatible with `--debug.` - -``--start`` will begin querying at the given timestamp. By default, ElastAlert will begin querying from the present. -Timestamp format is ``YYYY-MM-DDTHH-MM-SS[-/+HH:MM]`` (Note the T between date and hour). -Eg: ``--start 2014-09-26T12:00:00`` (UTC) or ``--start 2014-10-01T07:30:00-05:00`` - -``--end`` will cause ElastAlert to stop querying at the given timestamp. By default, ElastAlert will continue -to query indefinitely. - -``--rule`` will allow you to run only one rule. It must still be in the rules folder. -Eg: ``--rule this_rule.yaml`` - -``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory. - -``--prometheus_port`` exposes ElastAlert Prometheus metrics on the specified port. Prometheus metrics disabled by default. - -## Third Party Tools And Extras -### Kibana plugin -![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) -Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). - -### Docker -A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. - -```bash -git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 -p 3333:3333 \ - -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ - -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ - -v `pwd`/rules:/opt/elastalert/rules \ - -v `pwd`/rule_templates:/opt/elastalert/rule_templates \ - --net="host" \ - --name elastalert bitsensor/elastalert:latest -``` - -### ElastAlert-Docker - -This Dockerfile will build a Docker image for Elastalert. This image is compatible with the accompanying Helm chart for Kubernetes. - -https://github.com/jertel/elastalert-docker - -### ElastAlert uses ElastAlert Helm Chart almost as it is *Use original Docker Image to fix bugs - -[ElastAlert Server Helm Chart](https://github.com/daichi703n/elastalert-helm) - -[Praeco Helm Chart](https://github.com/daichi703n/praeco-helm) - -[Installing Praeco (ElastAlert GUI) into Kubernetes with Helm](https://en-designetwork.daichi703n.com/entry/2020/02/24/praeco-helm-kubernetes) - -### Praeco -![Praeco screenshot](https://user-images.githubusercontent.com/611996/47752071-7c4a9080-dc61-11e8-8ccf-2196f13429b2.png) -[Praeco](https://github.com/johnsusek/praeco) is a free open source GUI for ElastAlert. - -## Documentation - -Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). - -To build a html version of the docs locally - -``` -pip install sphinx_rtd_theme sphinx -cd docs -make html -``` - -View in browser at build/html/index.html - -## Configuration - -See config.yaml.example for details on configuration. - -## Example rules - -Examples of different types of rules can be found in example_rules/. - -- ``example_spike.yaml`` is an example of the "spike" rule type, which allows you to alert when the rate of events, averaged over a time period, -increases by a given factor. This example will send an email alert when there are 3 times more events matching a filter occurring within the -last 2 hours than the number of events in the previous 2 hours. - -- ``example_frequency.yaml`` is an example of the "frequency" rule type, which will alert when there are a given number of events occuring -within a time period. This example will send an email when 50 documents matching a given filter occur within a 4 hour timeframe. - -- ``example_change.yaml`` is an example of the "change" rule type, which will alert when a certain field in two documents changes. In this example, -the alert email is sent when two documents with the same 'username' field but a different value of the 'country_name' field occur within 24 hours -of each other. - -- ``example_new_term.yaml`` is an example of the "new term" rule type, which alerts when a new value appears in a field or fields. In this example, -an email is sent when a new value of ("username", "computer") is encountered in example login logs. - -## Frequently Asked Questions - -### My rule is not getting any hits? - -So you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like - -``` -filter: -- query: - query_string: - query: "foo: bar AND baz: abc*" -``` -If you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example, - -``` -filter: -- term: - foo: "Test Document" -``` - -will not match even if the original value for ``foo`` was exactly "Test Document". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening. - -### I got hits, why didn't I get an alert? - -If you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types. - -If you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs. - -If you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word "elastalert" as the "From:" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this. - -### Why did I only get one alert when I expected to get several? - -There is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use - -``` -realert: - minutes: 0 -``` - -You can of course set it higher as well. - -### How can I prevent duplicate alerts? - -By setting ``realert``, you will prevent the same rule from alerting twice in an amount of time. - -``` -realert: - days: 1 -``` - -You can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use - -``` -realert: - hours: 8 -query_key: user -``` - -Note that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use - -``` -query_key: [error, hostname] -``` - -Internally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``. - -The data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory. - -### How can I change what's in the alert? - -You can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only`` Or ``alert_text_type: alert_text_jinja``, it will be the entirety of the alert. You can also add different fields from the alert: - -With ``alert_text_type: alert_text_jinja`` by using [Jinja2](https://pypi.org/project/Jinja2/) Template. - -``` -alert_text_type: alert_text_jinja - -alert_text: | - Alert triggered! *({{num_hits}} Matches!)* - Something happened with {{username}} ({{email}}) - {{description|truncate}} - -``` - -> Top fields are accessible via `{{field_name}}` or `{{_data['field_name']}}`, `_data` is useful when accessing *fields with dots in their keys*, as Jinja treat dot as a nested field. -> If `_data` conflicts with your top level data, use ``jinja_root_name`` to change its name. - -With ``alert_text_type: alert_text_only`` by using Python style string formatting and ``alert_text_args``. For example - -``` -alert_text: "Something happened with {0} at {1}" -alert_text_type: alert_text_only -alert_text_args: ["username", "@timestamp"] -``` - -You can also limit the alert to only containing certain fields from the document by using ``include``. - -``` -include: ["ip_address", "hostname", "status"] -``` - -### My alert only contains data for one event, how can I see more? - -If you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have - -``` -top_count_keys: ["ip_address", "status"] -``` - -and 10 documents matched your alert, it may contain something like - -``` -ip_address: -127.0.0.1: 7 -10.0.0.1: 2 -192.168.0.1: 1 - -status: -200: 9 -500: 1 -``` - -### How can I make the alert come at a certain time? - -The ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using - -``` -aggregation: - schedule: '2 4 * * mon,fri' -``` - -### I have lots of documents and it's really slow, how can I speed it up? - -There are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index`` - -``` -index: logstash-%Y.%m -use_strftime_index: true -``` - -Another thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes. - -``` -buffer_time: - minutes: 5 -``` - -By default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp. - -If you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types. - -### Can I perform aggregations? - -The only aggregation supported currently is a terms aggregation, by setting ``use_terms_query``. - -### I'm not using @timestamp, what do I do? - -You can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents. - -### I'm using flatline but I don't see any alerts - -When using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them. - -### How can I get a "resolve" event? - -ElastAlert does not currently support stateful alerts or resolve events. - -### Can I set a warning threshold? - -Currently, the only way to set a warning threshold is by creating a second rule with a lower threshold. - -## License - -ElastAlert is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - -### Read the documentation at [Read the Docs](http://elastalert.readthedocs.org). - -### Questions? Drop by #elastalert on Freenode IRC. - -[0]: https://github.com/jertel/elastalert/blob/alt/README.md diff --git a/README.md b/README.md index 2539d382..70ea5231 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,10 @@ Elastalert 2 is backwards compatible with the original Elastalert rules. ## Documentation -Documentation for Elastalert 2 can be found on [readthedocs.com][3]. This is the place to start if you're not familiar with Elastalert at all. +Documentation, including an FAQ, for Elastalert 2 can be found on [readthedocs.com][3]. This is the place to start if you're not familiar with Elastalert at all. The full list of platforms that Elastalert can fire alerts into can be found [in the documentation][4]. -The original README for Elastalert can be found [here][5]. Please note that this file is -not being actively maintained, and will probably grow less accurate over time. ## Contributing @@ -25,17 +23,57 @@ The current status of the CI workflow: ## Docker -If you're interested in a pre-built Docker image for either the official yelp/elastalert release, or -for this fork, check out the [elastalert2][2] project on Docker Hub. +If you're interested in a pre-built Docker image check out the [elastalert2][2] project on Docker Hub. + +A properly configured elastalert_config.json file must be mounted into the container during startup of the container. Use the [example file][1] provided as a template, and once saved locally to a file such as `/tmp/elastalert.yaml`, run the container as follows: + +```bash +docker run -d -v /tmp/elastalert.yaml:/opt/config/elastalert_config.yaml jertel/elastalert2 +``` + +To build the image locally, install Docker and then run the following command: +``` +docker build . -t elastalert +``` + +## Kubernetes + +See the Helm chart [README.md](chart/elastalert/README.md) for information on installing this application into an existing Kubernetes cluster. + +## Releases + +As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][6] and let us know. + +Maintainers, when creating a new release, following the procedure below: + +1. Determine an appropriate new version number in the format _a.b.c_, using the following guidelines: + - The major version (a) should not change. + - The minor version (b) should be incremented if a new feature has been added or if a bug fix will have a significant user-impact. Reset the patch version to zero if the minor version is incremented. + - The patch version (c) should be incremented when low-impact bugs are fixed, or security vulnerabilities are patched. +2. Ensure the following are updated _before_ publishing/tagging the new release: + - [setup.py](setup.py): Match the version to the new release version + - [Chart.yaml](chart/elastalert2/Chart.yaml): Match chart version and the app version to the new release version (typically keep them in sync) + - [values.yaml](chart/elastalert2/values.yaml): Match the default image version to the new release version. + - [README.md](chart/elastalert2/README.md): Match the default image version to the new release version. +3. Double-check that the Docker image successfully built the latest image. +4. Create a [new][7] release. + - The title (and tag) of the release will be the same value as the new version determined in step 1. + - The description of the release will contain a bulleted list of all merged pull requests, in the following format: + `- PR/commit message #000 - @committer` + - Check the box to 'Create a discussion for this release'. + - Save the draft. +5. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. +6. Publish the release. ## License -Elastalert is licensed under the [Apache License, Version 2.0][6]. +Elastalert is licensed under the [Apache License, Version 2.0][5]. [0]: https://github.com/yelp/elastalert -[1]: https://github.com/jertel/elastalert +[1]: https://github.com/jertel/elastalert2/blob/master/config.yaml.example [2]: https://hub.docker.com/r/jertel/elastalert2 [3]: https://elastalert2.readthedocs.io/ [4]: https://elastalert2.readthedocs.io/en/latest/ruletypes.html#alerts -[5]: https://github.com/jertel/elastalert/blob/alt/README-old.md -[6]: http://www.apache.org/licenses/LICENSE-2 +[5]: http://www.apache.org/licenses/LICENSE-2 +[6]: https://github.com/jertel/elastalert2/discussions +[7]: https://github.com/jertel/elastalert2/releases/new \ No newline at end of file diff --git a/changelog.md b/changelog.md deleted file mode 100644 index 975d6855..00000000 --- a/changelog.md +++ /dev/null @@ -1,383 +0,0 @@ -# Change Log - -# v0.2.4 - -### Added -- Added back customFields support for The Hive - -# v0.2.3 - -### Added -- Added back TheHive alerter without TheHive4py library - -# v0.2.2 - -### Added -- Integration with Kibana Discover app -- Addied ability to specify opsgenie alert details  - -### Fixed -- Fix some encoding issues with command alerter -- Better error messages for missing config file -- Fixed an issue with run_every not applying per-rule -- Fixed an issue with rules not being removed -- Fixed an issue with top count keys and nested query keys -- Various documentation fixes -- Fixed an issue with not being able to use spike aggregation - -### Removed -- Remove The Hive alerter - -# v0.2.1 - -### Fixed -- Fixed an AttributeError introduced in 0.2.0 - -# v0.2.0 - -- Switched to Python 3 - -### Added -- Add rule loader class for customized rule loading -- Added thread based rules and limit_execution -- Run_every can now be customized per rule - -### Fixed -- Various small fixes - -# v0.1.39 - -### Added -- Added spike alerts for metric aggregations -- Allow SSL connections for Stomp -- Allow limits on alert text length -- Add optional min doc count for terms queries -- Add ability to index into arrays for alert_text_args, etc - -### Fixed -- Fixed bug involving --config flag with create-index -- Fixed some settings not being inherited from the config properly -- Some fixes for Hive alerter -- Close SMTP connections properly -- Fix timestamps in Pagerduty v2 payload -- Fixed an bug causing aggregated alerts to mix up - -# v0.1.38 - -### Added -- Added PagerTree alerter -- Added Line alerter -- Added more customizable logging -- Added new logic in test-rule to detemine the default timeframe - -### Fixed -- Fixed an issue causing buffer_time to sometimes be ignored - -# v0.1.37 - -### Added -- Added more options for Opsgenie alerter -- Added more pagerduty options -- Added ability to add metadata to elastalert logs - -### Fixed -- Fixed some documentation to be more clear -- Stop requiring doc_type for metric aggregations -- No longer puts quotes around regex terms in blacklists or whitelists - -# v0.1.36 - -### Added -- Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts -- Added option to skip Alerta certificate validation - -### Fixed -- Fixed a typo in the documentation for spike rule - -# v0.1.35 - -### Fixed -- Fixed an issue preventing new term rule from working with terms query - -# v0.1.34 - -### Added -- Added prefix/suffix support for summary table -- Added support for ignoring SSL validation in Slack -- More visible exceptions during query parse failures - -### Fixed -- Fixed top_count_keys when using compound query_key -- Fixed num_hits sometimes being reported too low -- Fixed an issue with setting ES_USERNAME via env -- Fixed an issue when using test script with custom timestamps -- Fixed a unicode error when using Telegram -- Fixed an issue with jsonschema version conflict -- Fixed an issue with nested timestamps in cardinality type - -# v0.1.33 - -### Added -- Added ability to pipe alert text to a command -- Add --start and --end support for elastalert-test-rule -- Added ability to turn blacklist/whitelist files into queries for better performance -- Allow setting of OpsGenie priority -- Add ability to query the adjacent index if timestamp_field not used for index timestamping -- Add support for pagerduty v2 -- Add option to turn off .raw/.keyword field postfixing in new term rule -- Added --use-downloaded feature for elastalert-test-rule - -### Fixed -- Fixed a bug that caused num_hits in matches to sometimes be erroneously small -- Fixed an issue with HTTP Post alerter that could cause it to hang indefinitely -- Fixed some issues with string formatting for various alerters -- Fixed a couple of incorrect parts of the documentation - -# v0.1.32 - -### Added -- Add support for setting ES url prefix via environment var -- Add support for using native Slack fields in alerts - -### Fixed -- Fixed a bug that would could scrolling queries to sometimes terminate early - -# v0.1.31 - -### Added -- Added ability to add start date to new term rule - -### Fixed -- Fixed a bug in create_index which would try to delete a nonexistent index -- Apply filters to new term rule all terms query -- Support Elasticsearch 6 for new term rule -- Fixed is_enabled not working on rule changes - - -# v0.1.30 - -### Added -- Alerta alerter -- Added support for transitioning JIRA issues -- Option to recreate index in elastalert-create-index - -### Fixed -- Update jira_ custom fields before each alert if they were modified -- Use json instead of simplejson -- Allow for relative path for smtp_auth_file -- Fixed some grammar issues -- Better code formatting of index mappings -- Better formatting and size limit for HipChat HTML -- Fixed gif link in readme for kibana plugin -- Fixed elastalert-test-rule with Elasticsearch > 4 -- Added documentation for is_enabled option - -## v0.1.29 - -### Added -- Added a feature forget_keys to prevent realerting when using flatline with query_key -- Added a new alert_text_type, aggregation_summary_only - -### Fixed -- Fixed incorrect documentation about es_conn_timeout default - -## v0.1.28 - -### Added -- Added support for Stride formatting of simple HTML tags -- Added support for custom titles in Opsgenie alerts -- Added a denominator to percentage match based alerts - -### Fixed -- Fixed a bug with Stomp alerter connections -- Removed escaping of some characaters in Slack messages - -## v0.1.27 - -# Added -- Added support for a value other than in formatted alerts - -### Fixed -- Fixed a failed creation of elastalert indicies when using Elasticsearch 6 -- Truncate Telegram alerts to avoid API errors - -## v0.1.26 - -### Added -- Added support for Elasticsearch 6 -- Added support for mentions in Hipchat - -### Fixed -- Fixed an issue where a nested field lookup would crash if one of the intermediate fields was null - -## v0.1.25 - -### Fixed -- Fixed a bug causing new term rule to break unless you passed a start time -- Add a slight clarification on the localhost:9200 reported in es_debug_trace - -## v0.1.24 - -### Fixed -- Pinned pytest -- create-index reads index name from config.yaml -- top_count_keys now works for context on a flatline rule type -- Fixed JIRA behavior for issues with statuses that have spaces in the name - -## v0.1.22 - -### Added -- Added Stride alerter -- Allow custom string formatters for aggregation percentage -- Added a field to disable rules from config -- Added support for subaggregations for the metric rule type - -### Fixed -- Fixed a bug causing create-index to fail if missing config.yaml -- Fixed a bug when using ES5 with query_key and top_count_keys -- Allow enhancements to set and clear arbitrary JIRA fields -- Fixed a bug causing timestamps to be formatted in scientific notation -- Stop attempting to initialize alerters in debug mode -- Changed default alert ordering so that JIRA tickets end up in other alerts -- Fixed a bug when using Stomp alerter with complex query_key -- Fixed a bug preventing hipchat room ID from being an integer -- Fixed a bug causing duplicate alerts when using spike with alert_on_new_data -- Minor fixes to summary table formatting -- Fixed elastalert-test-rule when using new term rule type - -## v0.1.21 - -### Fixed -- Fixed an incomplete bug fix for preventing duplicate enhancement runs - -## v0.1.20 - -### Added -- Added support for client TLS keys - -### Fixed -- Fixed the formatting of summary tables in Slack -- Fixed ES_USE_SSL env variable -- Fixed the unique value count printed by new_term rule type -- Jira alerter no longer uses the non-existent json code formatter - -## v0.1.19 - -### Added -- Added support for populating JIRA fields via fields in the match -- Added support for using a TLS certificate file for SMTP connections -- Allow a custom suffix for non-analyzed Elasticsearch fields, like ".raw" or ".keyword" -- Added match_time to Elastalert alert documents in Elasticsearch - -### Fixed -- Fixed an error in the documentation for rule importing -- Prevent enhancements from re-running on retried alerts -- Fixed a bug when using custom timestamp formats and new term rule -- Lowered jira_bump_after_inactivity default to 0 days - -## v0.1.18 - -### Added -- Added a new alerter "post" based on "simple" which makes POSTS JSON to HTTP endpoints -- Added an option jira_bump_after_inacitivty to prevent ElastAlert commenting on active JIRA tickets - -### Removed -- Removed "simple" alerter, replaced by "post" - -## v0.1.17 - -### Added -- Added a --patience flag to allow Elastalert to wait for Elasticsearch to become available -- Allow custom PagerDuty alert titles via alert_subject - -## v0.1.16 - -### Fixed -- Fixed a bug where JIRA titles might not use query_key values -- Fixed a bug where flatline alerts don't respect query_key for realert -- Fixed a typo "twilio_accout_sid" - -### Added -- Added support for env variables in kibana4 dashboard links -- Added ca_certs option for custom CA support - -## v0.1.15 - -### Fixed -- Fixed a bug where Elastalert would crash on connection error during startup -- Fixed some typos in documentation -- Fixed a bug in metric bucket offset calculation -- Fixed a TypeError in Service Now alerter - -### Added -- Added support for compound compare key in change rules -- Added support for absolute paths in rule config imports -- Added Microsoft Teams alerter -- Added support for markdown in Slack alerts -- Added error codes to test script -- Added support for lists in email_from_field - - -## v0.1.14 - 2017-05-11 - -### Fixed -- Twilio alerter uses the from number appropriately -- Fixed a TypeError in SNS alerter -- Some changes to requirements.txt and setup.py -- Fixed a TypeError in new term rule - -### Added -- Set a custom pagerduty incident key -- Preserve traceback in most exceptions - -## v0.1.12 - 2017-04-21 - -### Fixed -- Fixed a bug causing filters to be ignored when using Elasticsearch 5 - - -## v0.1.11 - 2017-04-19 - -### Fixed -- Fixed an issue that would cause filters starting with "query" to sometimes throw errors in ES5 -- Fixed a bug with multiple versions of ES on different rules -- Fixed a possible KeyError when using use_terms_query with ES5 - -## v0.1.10 - 2017-04-17 - -### Fixed -- Fixed an AttributeError occuring with older versions of Elasticsearch library -- Made example rules more consistent and with unique names -- Fixed an error caused by a typo when es_username is used - -## v0.1.9 - 2017-04-14 - -### Added -- Added a changelog -- Added metric aggregation rule type -- Added percentage match rule type -- Added default doc style and improved the instructions -- Rule names will default to the filename -- Added import keyword in rules to include sections from other files -- Added email_from_field option to derive the recipient from a field in the match -- Added simple HTTP alerter -- Added Exotel SMS alerter -- Added a readme link to third party Kibana plugin -- Added option to use env variables to configure some settings -- Added duplicate hits count in log line - -### Fixed -- Fixed a bug in change rule where a boolean false would be ignored -- Clarify documentation on format of alert_text_args and alert_text_kw -- Fixed a bug preventing new silence stashes from being loaded after a rule has previous alerted -- Changed the default es_host in elastalert-test-rule to localhost -- Fixed a bug preventing ES <5.0 formatted queries working in elastalert-test-rule -- Fixed top_count_keys adding .raw on ES >5.0, uses .keyword instead -- Fixed a bug causing compound aggregation keys not to work -- Better error reporting for the Jira alerter -- AWS request signing now refreshes credentials, uses boto3 -- Support multiple ES versions on different rules -- Added documentation for percentage match rule type - -### Removed -- Removed a feature that would disable writeback_es on errors, causing various issues diff --git a/chart/elastalert2/.helmignore b/chart/elastalert2/.helmignore new file mode 100644 index 00000000..f0c13194 --- /dev/null +++ b/chart/elastalert2/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml new file mode 100644 index 00000000..813b4808 --- /dev/null +++ b/chart/elastalert2/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +description: Elastalert 2 is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. +name: elastalert2 +version: 2.0.2 +appVersion: 2.0.2 +home: https://github.com/jertel/elastalert2 +sources: +- https://github.com/jertel/elastalert2 +maintainers: + - name: jertel + email: jertel@codesim.com +engine: gotpl diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md new file mode 100644 index 00000000..72173672 --- /dev/null +++ b/chart/elastalert2/README.md @@ -0,0 +1,90 @@ + +# Elastalert 2 Helm Chart for Kubernetes + +An Elastalert 2 helm chart is available in the jertel Helm repository, and can be installed into an existing Kubernetes cluster by following the instructions below. + +## Installing the Chart + +Add the jertel repository to your Helm configuration: + +```console +helm repo add codesim https://helm.jertel.com +``` + +Next, install the chart with a release name, such as _elastalert2_: + +```console +helm install elastalert2 jertel/elastalert2 +``` + +The command deploys Elastalert 2 on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +See the comment in the default `values.yaml` for specifying a `writebackIndex` for ES 5.x. + +If necessary, open Dev Tools on Kibana and send the below request to avoid errors like `RequestError: TransportError(400, u'search_phase_execution_exception', u'No mapping found for [alert_time] in order to sort on')` + +``` +PUT /elastalert/_mapping/elastalert +{ + "properties": { + "alert_time": {"type": "date"} + } +} +``` + +## Uninstalling the Chart + +To uninstall/delete the Elastalert 2 deployment: + +```console +helm delete elastalert2 --purge +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|---------------------------------| +| `image.repository` | docker image | jertel/elastalert-docker | +| `image.tag` | docker image tag | 2,0.2 | +| `image.pullPolicy` | image pull policy | IfNotPresent | +| `podAnnotations` | Annotations to be added to pods | {} | +| `command` | command override for container | `NULL` | +| `args` | args override for container | `NULL` | +| `replicaCount` | number of replicas to run | 1 | +| `elasticsearch.host` | elasticsearch endpoint to use | elasticsearch | +| `elasticsearch.port` | elasticsearch port to use | 9200 | +| `elasticsearch.useSsl` | whether or not to connect to es_host using SSL | False | +| `elasticsearch.username` | Username for ES with basic auth | `NULL` | +| `elasticsearch.password` | Password for ES with basic auth | `NULL` | +| `elasticsearch.credentialsSecret` | Specifies an existing secret to be used for the ES username/password auth | `NULL` | +| `elasticsearch.credentialsSecretUsernameKey` | The key in elasticsearch.credentialsSecret that stores the ES password auth | `NULL` | +| `elasticsearch.credentialsSecretPasswordKey` | The key in elasticsearch.credentialsSecret that stores the ES username auth | `NULL` | +| `elasticsearch.verifyCerts` | whether or not to verify TLS certificates | True | +| `elasticsearch.clientCert` | path to a PEM certificate to use as the client certificate | /certs/client.pem | +| `elasticsearch.clientKey` | path to a private key file to use as the client key | /certs/client-key.pem | +| `elasticsearch.caCerts` | path to a CA cert bundle to use to verify SSL connections | /certs/ca.pem | +| `elasticsearch.certsVolumes` | certs volumes, required to mount ssl certificates when elasticsearch has tls enabled | `NULL` | +| `elasticsearch.certsVolumeMounts` | mount certs volumes, required to mount ssl certificates when elasticsearch has tls enabled | `NULL` | +| `extraConfigOptions` | Additional options to propagate to all rules, cannot be `alert`, `type`, `name` or `index` | `{}` | +| `secretConfigName` | name of the secret which holds the Elastalert config. **Note:** this will completely overwrite the generated config | `NULL` | +| `secretRulesName` | name of the secret which holds the Elastalert rules. **Note:** this will overwrite the generated rules | `NULL` | +| `secretRulesList` | a list of rules to enable from the secret | [] | +| `optEnv` | Additional pod environment variable definitions | [] | +| `extraVolumes` | Additional volume definitions | [] | +| `extraVolumeMounts` | Additional volumeMount definitions | [] | +| `serviceAccount.create` | Specifies whether a service account should be created. | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `podSecurityPolicy.create` | Create pod security policy resources | `false` | +| `resources` | Container resource requests and limits | {} | +| `rules` | Rule and alert configuration for Elastalert | {} example shown in values.yaml | +| `runIntervalMins` | Default interval between alert checks, in minutes | 1 | +| `realertIntervalMins` | Time between alarms for same rule, in minutes | `NULL` | +| `alertRetryLimitMins` | Time to retry failed alert deliveries, in minutes | 2880 (2 days) | +| `bufferTimeMins` | Default rule buffer time, in minutes | 15 | +| `writebackIndex` | Name or prefix of elastalert index(es) | elastalert | +| `nodeSelector` | Node selector for deployment | {} | +| `affinity` | Affinity specifications for the deployed pod(s) | {} | +| `tolerations` | Tolerations for deployment | [] | diff --git a/chart/elastalert2/templates/NOTES.txt b/chart/elastalert2/templates/NOTES.txt new file mode 100644 index 00000000..7b1c2985 --- /dev/null +++ b/chart/elastalert2/templates/NOTES.txt @@ -0,0 +1 @@ +1. Elastalert is now running against: {{ .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }} \ No newline at end of file diff --git a/chart/elastalert2/templates/_helpers.tpl b/chart/elastalert2/templates/_helpers.tpl new file mode 100644 index 00000000..2fbdad46 --- /dev/null +++ b/chart/elastalert2/templates/_helpers.tpl @@ -0,0 +1,36 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "elastalert.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "elastalert.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "elastalert.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "elastalert.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/chart/elastalert2/templates/deployment.yaml b/chart/elastalert2/templates/deployment.yaml new file mode 100644 index 00000000..0d696a12 --- /dev/null +++ b/chart/elastalert2/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "elastalert.fullname" . }} + labels: + app: {{ template "elastalert.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ template "elastalert.name" . }} + release: {{ .Release.Name }} + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} + checksum/rules: {{ include (print $.Template.BasePath "/rules.yaml") . | sha256sum }} +{{- if .Values.podAnnotations }} +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + labels: + name: {{ template "elastalert.fullname" . }}-elastalert + app: {{ template "elastalert.name" . }} + release: {{ .Release.Name }} + spec: + serviceAccountName: {{ include "elastalert.serviceAccountName" . }} + containers: + - name: elastalert + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} +{{- if .Values.command }} + command: +{{ toYaml .Values.command | indent 10 }} +{{- end }} +{{- if .Values.args }} + args: +{{ toYaml .Values.args | indent 10 }} +{{- end }} + volumeMounts: + - name: config + mountPath: '/opt/config' + - name: rules + mountPath: '/opt/rules' +{{- if .Values.elasticsearch.certsVolumeMounts }} +{{ toYaml .Values.elasticsearch.certsVolumeMounts | indent 10 }} +{{- end }} +{{- if .Values.extraVolumeMounts }} +{{ toYaml .Values.extraVolumeMounts | indent 10 }} +{{- end }} + resources: +{{ toYaml .Values.resources | indent 12 }} + env: +{{- if .Values.elasticsearch.credentialsSecret }} +{{- if .Values.elasticsearch.credentialsSecretUsernameKey }} + - name: ES_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.elasticsearch.credentialsSecret }} + key: {{ .Values.elasticsearch.credentialsSecretUsernameKey }} +{{- end }} +{{- if .Values.elasticsearch.credentialsSecretPasswordKey }} + - name: ES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.elasticsearch.credentialsSecret }} + key: {{ .Values.elasticsearch.credentialsSecretPasswordKey }} +{{- end }} +{{- end }} +{{- if .Values.optEnv }} +{{ .Values.optEnv | toYaml | indent 10}} +{{- end }} + restartPolicy: Always +{{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} + volumes: + - name: rules +{{- if .Values.secretRulesName }} + secret: + secretName: {{ .Values.secretRulesName }} + items: +{{- range $key := .Values.secretRulesList }} + - key: {{ $key }} + path: {{ $key}}.yaml +{{- end }} +{{- else }} + configMap: + name: {{ template "elastalert.fullname" . }}-rules + items: +{{- range $key, $value := .Values.rules }} + - key: {{ $key }} + path: {{ $key}}.yaml +{{- end }} +{{- end }} + - name: config +{{- if .Values.secretConfigName }} + secret: + secretName: {{ .Values.secretConfigName }} +{{- else }} + configMap: + name: {{ template "elastalert.fullname" . }}-config +{{- end }} + items: + - key: elastalert_config + path: elastalert_config.yaml +{{- if .Values.elasticsearch.certsVolumes }} +{{ toYaml .Values.elasticsearch.certsVolumes | indent 8 }} +{{- end }} +{{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | indent 8 }} +{{- end }} diff --git a/chart/elastalert2/templates/podsecuritypolicy.yaml b/chart/elastalert2/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000..e3777203 --- /dev/null +++ b/chart/elastalert2/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if .Values.podSecurityPolicy.create }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "elastalert.fullname" . }} + labels: + app: {{ template "elastalert.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + # Prevents running in privileged mode + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + volumes: + - configMap + - secret + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: MustRunAs + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: MustRunAs + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/chart/elastalert2/templates/role.yaml b/chart/elastalert2/templates/role.yaml new file mode 100644 index 00000000..93b9cadd --- /dev/null +++ b/chart/elastalert2/templates/role.yaml @@ -0,0 +1,20 @@ +{{- if .Values.podSecurityPolicy.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "elastalert.fullname" . }} + labels: + app: {{ template "elastalert.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - {{ template "elastalert.fullname" . }} + verbs: + - use +{{- end -}} diff --git a/chart/elastalert2/templates/rolebinding.yaml b/chart/elastalert2/templates/rolebinding.yaml new file mode 100644 index 00000000..67a69d1f --- /dev/null +++ b/chart/elastalert2/templates/rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podSecurityPolicy.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "elastalert.fullname" . }} + labels: + app: {{ template "elastalert.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "elastalert.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "elastalert.serviceAccountName" . }} +{{- end -}} diff --git a/chart/elastalert2/templates/rules.yaml b/chart/elastalert2/templates/rules.yaml new file mode 100644 index 00000000..1e4afd45 --- /dev/null +++ b/chart/elastalert2/templates/rules.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "elastalert.fullname" . }}-rules + labels: + app: {{ template "elastalert.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: +{{- range $key, $value := .Values.rules }} +{{ $key | indent 2}}: |- +{{ $value | indent 4}} +{{- end }} diff --git a/chart/elastalert2/templates/serviceaccount.yaml b/chart/elastalert2/templates/serviceaccount.yaml new file mode 100644 index 00000000..dc1e08c5 --- /dev/null +++ b/chart/elastalert2/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "elastalert.serviceAccountName" . }} + labels: + app: {{ template "elastalert.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/chart/elastalert2/values.yaml b/chart/elastalert2/values.yaml new file mode 100644 index 00000000..c3a24acb --- /dev/null +++ b/chart/elastalert2/values.yaml @@ -0,0 +1,228 @@ +# number of replicas to run +replicaCount: 1 + +# number of helm release revisions to retain +revisionHistoryLimit: 5 + +# Default internal between alert checks against the elasticsearch datasource, in minutes +runIntervalMins: 1 + +# Default rule buffer duration, in minutes +bufferTimeMins: 15 + +# Amount of time to retry and deliver failed alerts (1440 minutes per day) +alertRetryLimitMins: 2880 + +# Default time before realerting, in minutes +realertIntervalMins: "" + +# For ES 5: The name of the index which stores elastalert 2 statuses, typically elastalert_status +# For ES 6: The prefix of the names of indices which store elastalert 2 statuses, typically elastalert +# +writebackIndex: elastalert + +image: + # docker image + repository: jertel/elastalert2 + # docker image tag + tag: 2.0.2 + pullPolicy: IfNotPresent + +resources: {} + +# Annotations to be added to pods +podAnnotations: {} + +elasticsearch: + # elasticsearch endpoint e.g. (svc.namespace||svc) + host: elasticsearch + # elasticsearch port + port: 9200 + # whether or not to connect to es_host using TLS + useSsl: "False" + # Username if authenticating to ES with basic auth + username: "" + # Password if authenticating to ES with basic auth + password: "" + # Specifies an existing secret to be used for the ES username/password + credentialsSecret: "" + # The key in elasticsearch.credentialsSecret that stores the ES password + credentialsSecretUsernameKey: "" + # The key in elasticsearch.credentialsSecret that stores the ES username + credentialsSecretPasswordKey: "" + # whether or not to verify TLS certificates + verifyCerts: "True" + # Enable certificate based authentication + # path to a PEM certificate to use as the client certificate + # clientCert: "/certs/client.pem" + # path to a private key file to use as the client key + # clientKey: "/certs/client-key.pem" + # path to a CA cert bundle to use to verify SSL connections + # caCerts: "/certs/ca.pem" + # # certs volumes, required to mount ssl certificates when elasticsearch has tls enabled + # certsVolumes: + # - name: es-certs + # secret: + # defaultMode: 420 + # secretName: es-certs + # # mount certs volumes, required to mount ssl certificates when elasticsearch has tls enabled + # certsVolumeMounts: + # - name: es-certs + # mountPath: /certs + # readOnly: true + +# Optional env variables for the pod +optEnv: [] + +extraConfigOptions: {} + # # Options to propagate to all rules, e.g. a common slack_webhook_url or kibana_url + # # Please note at the time of implementing this value, it will not work for required_locals + # # Which MUST be set at the rule level, these are: ['alert', 'type', 'name', 'index'] + # generate_kibana_link: true + # kibana_url: https://kibana.yourdomain.com + # slack_webhook_url: dummy + +# To load Elastalert 2 config via secret, uncomment the line below +# secretConfigName: elastalert-config-secret + +# Example of a secret config + +#apiVersion: v1 +#kind: Secret +#metadata: +# name: elastalert-config-secret +#type: Opaque +#stringData: +# elastalert_config: |- +# rules_folder: /opt/rules +# scan_subdirectories: false +# run_every: +# minutes: 1 +# buffer_time: +# minutes: 15 +# es_host: elasticsearch +# es_port: 9200 +# writeback_index: elastalert +# use_ssl: False +# verify_certs: True +# alert_time_limit: +# minutes: 2880 +# slack_webhook_url: https://hooks.slack.com/services/xxxx +# slack_channel_override: '#alerts' + + +# To load Elastalert's rules via secret, uncomment the line below +#secretRulesName: elastalert-rules-secret + +# Additionally, you must specificy which rules to load from the secret +#secretRulesList: [ "rule_1", "rule_2" ] + +# Example of secret rules + +#apiVersion: v1 +#kind: Secret +#metadata: +# name: elastalert-rules-secret +# namespace: elastic-system +#type: Opaque +#stringData: +# rule_1: |- +# name: Rule 1 +# type: frequency +# index: index1-* +# num_events: 3 +# timeframe: +# minutes: 1 +# alert: +# - "slack" +# rule_2: |- +# name: Rule 2 +# type: frequency +# index: index2-* +# num_events: 5 +# timeframe: +# minutes: 10 +# alert: +# - "slack" + +# Command and args override for container e.g. (https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/) +# command: ["YOUR_CUSTOM_COMMAND"] +# args: ["YOUR", "CUSTOM", "ARGS"] + +# rule configurations e.g. (http://elastalert2.readthedocs.io/en/latest/) +rules: {} + # deadman_slack: |- + # --- + # name: Deadman Switch Slack + # type: frequency + # index: containers-* + # num_events: 3 + # timeframe: + # minutes: 3 + # filter: + # - term: + # message: "deadmanslack" + # alert: + # - "slack" + # slack: + # slack_webhook_url: dummy + # deadman_pagerduty: |- + # --- + # name: Deadman Switch PagerDuty + # type: frequency + # index: containers-* + # num_events: 3 + # timeframe: + # minutes: 3 + # filter: + # - term: + # message: "deadmanpd" + # alert: + # - "pagerduty" + # pagerduty: + # pagerduty_service_key: dummy + # pagerduty_client_name: Elastalert Deadman Switch + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +# Enable pod security policy +# https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + create: false + +# Support using node selectors and tolerations +# nodeSelector: +# "node-role.kubernetes.io/infra_worker": "true" +nodeSelector: {} + +# Specify node affinity or anti-affinity specifications +affinity: {} + +# tolerations: +# - key: "node_role" +# operator: "Equal" +# value: "infra_worker" +# effect: "NoSchedule" +tolerations: [] + +extraVolumes: [] + # - name: smtp-auth + # secret: + # secretName: elastalert-smtp-auth + # items: + # - key: smtp_auth.yaml + # path: smtp_auth.yaml + # mode: 0400 + +extraVolumeMounts: [] + # - name: smtp-auth + # mountPath: /opt/config-smtp/smtp_auth.yaml + # subPath: smtp_auth.yaml + # readOnly: true diff --git a/setup.py b/setup.py index 5abdd436..058ca531 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,13 @@ base_dir = os.path.dirname(__file__) setup( - name='elastalert', - version='0.2.4', + name='elastalert2', + version='2.0.2', description='Runs custom filters on Elasticsearch and alerts on matches', - author='Quentin Long', - author_email='qlo@yelp.com', setup_requires='setuptools', - license='Copyright 2014 Yelp', + license='Apache 2.0', classifiers=[ - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], From 358a11c7ff7f6e6aa2a04a1298c13645b8527d75 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:30:20 -0400 Subject: [PATCH 115/128] Fix typographical errors. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 70ea5231..b899d3a5 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,13 @@ docker build . -t elastalert ## Kubernetes -See the Helm chart [README.md](chart/elastalert/README.md) for information on installing this application into an existing Kubernetes cluster. +See the Helm chart [README.md](chart/elastalert2/README.md) for information on installing this application into an existing Kubernetes cluster. ## Releases As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][6] and let us know. -Maintainers, when creating a new release, following the procedure below: +Maintainers, when creating a new release, follow the procedure below: 1. Determine an appropriate new version number in the format _a.b.c_, using the following guidelines: - The major version (a) should not change. From 57ba69e45c7c4773d287538ca554001722b702cd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:51:08 -0400 Subject: [PATCH 116/128] Add new issue template to help prevent unnecessary issues from being created. --- .github/ISSUE_TEMPLATE | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE new file mode 100644 index 00000000..0c580a74 --- /dev/null +++ b/.github/ISSUE_TEMPLATE @@ -0,0 +1,3 @@ +PLEASE READ: The majority of topics are better suited for the [Discussion forum](https://github.com/jertel/elastalert2/discussions). Please search the discussions area first, for keywords that could be associated with the problem you are experiencing. If you do not see an existing discussion, please open a new discussion and include sufficient details for someone in the community to help you. + +If you are confident you have discovered a legitimate issue, attach logs and reproduction steps to this issue. Failure to provide sufficient information will likely cause this issue to go stale and eventually be deleted. \ No newline at end of file From 0fc105e1fc91126098026398acdcb54cb5694fc2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:52:02 -0400 Subject: [PATCH 117/128] Add new issue template to help prevent unnecessary issues from being created. --- .github/ISSUE_TEMPLATE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index 0c580a74..81f621e0 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -1,3 +1,3 @@ -PLEASE READ: The majority of topics are better suited for the [Discussion forum](https://github.com/jertel/elastalert2/discussions). Please search the discussions area first, for keywords that could be associated with the problem you are experiencing. If you do not see an existing discussion, please open a new discussion and include sufficient details for someone in the community to help you. +PLEASE READ: The majority of topics are better suited for the Discussion forum. You can access this area by clicking The Discussions link above. Please search the discussions area first, for keywords that could be associated with the problem you are experiencing. If you do not see an existing discussion, please open a new discussion and include sufficient details for someone in the community to help you. If you are confident you have discovered a legitimate issue, attach logs and reproduction steps to this issue. Failure to provide sufficient information will likely cause this issue to go stale and eventually be deleted. \ No newline at end of file From 592d226550d222e3793de3a006db7afa7297fcc5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:54:23 -0400 Subject: [PATCH 118/128] Correct helm repo instructions --- chart/elastalert2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index 72173672..a6bbc175 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -8,7 +8,7 @@ An Elastalert 2 helm chart is available in the jertel Helm repository, and can b Add the jertel repository to your Helm configuration: ```console -helm repo add codesim https://helm.jertel.com +helm repo add jertel https://helm.jertel.com ``` Next, install the chart with a release name, such as _elastalert2_: From 9f7f4706abb49e6b90a7dda9a59072b802fca62e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 14:56:05 -0400 Subject: [PATCH 119/128] Correct version in Helm README --- chart/elastalert2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index a6bbc175..a3dd5c91 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -47,7 +47,7 @@ The command removes all the Kubernetes components associated with the chart and | Parameter | Description | Default | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|---------------------------------| | `image.repository` | docker image | jertel/elastalert-docker | -| `image.tag` | docker image tag | 2,0.2 | +| `image.tag` | docker image tag | 2.0.2 | | `image.pullPolicy` | image pull policy | IfNotPresent | | `podAnnotations` | Annotations to be added to pods | {} | | `command` | command override for container | `NULL` | From 6e1eba21564cd12392e59dfbfa76e044b22dee31 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:08:29 -0400 Subject: [PATCH 120/128] Create python-publish.yml --- .github/workflows/python-publish.yml | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..8c173da2 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 8a554265669e803ab2e9f75ad7efb0b1f8df4973 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:17:07 -0400 Subject: [PATCH 121/128] Update release procedure and scripts --- .github/workflows/{release.yml => upload_chart.yml} | 4 ++-- README.md | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) rename .github/workflows/{release.yml => upload_chart.yml} (92%) diff --git a/.github/workflows/release.yml b/.github/workflows/upload_chart.yml similarity index 92% rename from .github/workflows/release.yml rename to .github/workflows/upload_chart.yml index 5f51356b..7cba508b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/upload_chart.yml @@ -1,8 +1,8 @@ -name: upload-chart +name: upload_chart on: release: - types: [published] + types: [created] jobs: build: diff --git a/README.md b/README.md index b899d3a5..b55fcd88 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,12 @@ Maintainers, when creating a new release, follow the procedure below: `- PR/commit message #000 - @committer` - Check the box to 'Create a discussion for this release'. - Save the draft. -5. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. -6. Publish the release. +5. Verify that artifacts have been published: + - Python PIP package was [published][9] successfully. + - Helm chart has been [published][10] successfully. + - Docker Hub image was [published][8] successfully. +6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. +7. Publish the release. ## License @@ -76,4 +80,7 @@ Elastalert is licensed under the [Apache License, Version 2.0][5]. [4]: https://elastalert2.readthedocs.io/en/latest/ruletypes.html#alerts [5]: http://www.apache.org/licenses/LICENSE-2 [6]: https://github.com/jertel/elastalert2/discussions -[7]: https://github.com/jertel/elastalert2/releases/new \ No newline at end of file +[7]: https://github.com/jertel/elastalert2/releases/new +[8]: https://hub.docker.com/r/jertel/elastalert2/builds +[9]: https://github.com/jertel/elastalert2/actions/workflows/python-publish.yml +[10]: https://github.com/jertel/elastalert2/actions/workflows/upload_chart.yml \ No newline at end of file From aa6bbf5516e1586636247629f08336c3ef86a165 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:18:00 -0400 Subject: [PATCH 122/128] Update release procedure and scripts --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 8c173da2..0b003bac 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,7 +1,7 @@ # This workflow will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries -name: Upload Python Package +name: upload_python_package on: release: From 3778040d3f1f1c75aaeeb680f9d1bf87d3f2d96b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:26:13 -0400 Subject: [PATCH 123/128] Split out release procedure from README into its own file --- README.md | 29 ----------------------------- RELEASE.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 29 deletions(-) create mode 100644 RELEASE.md diff --git a/README.md b/README.md index b55fcd88..c8fa89e1 100644 --- a/README.md +++ b/README.md @@ -40,35 +40,6 @@ docker build . -t elastalert See the Helm chart [README.md](chart/elastalert2/README.md) for information on installing this application into an existing Kubernetes cluster. -## Releases - -As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][6] and let us know. - -Maintainers, when creating a new release, follow the procedure below: - -1. Determine an appropriate new version number in the format _a.b.c_, using the following guidelines: - - The major version (a) should not change. - - The minor version (b) should be incremented if a new feature has been added or if a bug fix will have a significant user-impact. Reset the patch version to zero if the minor version is incremented. - - The patch version (c) should be incremented when low-impact bugs are fixed, or security vulnerabilities are patched. -2. Ensure the following are updated _before_ publishing/tagging the new release: - - [setup.py](setup.py): Match the version to the new release version - - [Chart.yaml](chart/elastalert2/Chart.yaml): Match chart version and the app version to the new release version (typically keep them in sync) - - [values.yaml](chart/elastalert2/values.yaml): Match the default image version to the new release version. - - [README.md](chart/elastalert2/README.md): Match the default image version to the new release version. -3. Double-check that the Docker image successfully built the latest image. -4. Create a [new][7] release. - - The title (and tag) of the release will be the same value as the new version determined in step 1. - - The description of the release will contain a bulleted list of all merged pull requests, in the following format: - `- PR/commit message #000 - @committer` - - Check the box to 'Create a discussion for this release'. - - Save the draft. -5. Verify that artifacts have been published: - - Python PIP package was [published][9] successfully. - - Helm chart has been [published][10] successfully. - - Docker Hub image was [published][8] successfully. -6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. -7. Publish the release. - ## License Elastalert is licensed under the [Apache License, Version 2.0][5]. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..c43ee69b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,28 @@ +# Releases + +As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][6] and let us know. + +Maintainers, when creating a new release, follow the procedure below: + +1. Determine an appropriate new version number in the format _a.b.c_, using the following guidelines: + - The major version (a) should not change. + - The minor version (b) should be incremented if a new feature has been added or if a bug fix will have a significant user-impact. Reset the patch version to zero if the minor version is incremented. + - The patch version (c) should be incremented when low-impact bugs are fixed, or security vulnerabilities are patched. +2. Ensure the following are updated _before_ publishing/tagging the new release: + - [setup.py](setup.py): Match the version to the new release version + - [Chart.yaml](chart/elastalert2/Chart.yaml): Match chart version and the app version to the new release version (typically keep them in sync) + - [values.yaml](chart/elastalert2/values.yaml): Match the default image version to the new release version. + - [README.md](chart/elastalert2/README.md): Match the default image version to the new release version. +3. Double-check that the Docker image successfully built the latest image. +4. Create a [new][7] release. + - The title (and tag) of the release will be the same value as the new version determined in step 1. + - The description of the release will contain a bulleted list of all merged pull requests, in the following format: + `- PR/commit message #000 - @committer` + - Check the box to 'Create a discussion for this release'. + - Save the draft. +5. Verify that artifacts have been published: + - Python PIP package was [published][9] successfully. + - Helm chart has been [published][10] successfully. + - Docker Hub image was [published][8] successfully. +6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. +7. Publish the release. \ No newline at end of file From df805fc32a7db45d14fc86dacfce2193d23963ec Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:27:10 -0400 Subject: [PATCH 124/128] Split out release procedure from README into its own file --- README.md | 3 --- RELEASE.md | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c8fa89e1..f06ccb26 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,3 @@ Elastalert is licensed under the [Apache License, Version 2.0][5]. [5]: http://www.apache.org/licenses/LICENSE-2 [6]: https://github.com/jertel/elastalert2/discussions [7]: https://github.com/jertel/elastalert2/releases/new -[8]: https://hub.docker.com/r/jertel/elastalert2/builds -[9]: https://github.com/jertel/elastalert2/actions/workflows/python-publish.yml -[10]: https://github.com/jertel/elastalert2/actions/workflows/upload_chart.yml \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md index c43ee69b..0323a852 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -25,4 +25,8 @@ Maintainers, when creating a new release, follow the procedure below: - Helm chart has been [published][10] successfully. - Docker Hub image was [published][8] successfully. 6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. -7. Publish the release. \ No newline at end of file +7. Publish the release. + +[8]: https://hub.docker.com/r/jertel/elastalert2/builds +[9]: https://github.com/jertel/elastalert2/actions/workflows/python-publish.yml +[10]: https://github.com/jertel/elastalert2/actions/workflows/upload_chart.yml \ No newline at end of file From 4180fd43ad57f119513c49c2e906336480a7e052 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:29:14 -0400 Subject: [PATCH 125/128] Split out release procedure from README into its own file --- README.md | 3 +-- RELEASE.md | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f06ccb26..463b238b 100644 --- a/README.md +++ b/README.md @@ -50,5 +50,4 @@ Elastalert is licensed under the [Apache License, Version 2.0][5]. [3]: https://elastalert2.readthedocs.io/ [4]: https://elastalert2.readthedocs.io/en/latest/ruletypes.html#alerts [5]: http://www.apache.org/licenses/LICENSE-2 -[6]: https://github.com/jertel/elastalert2/discussions -[7]: https://github.com/jertel/elastalert2/releases/new + diff --git a/RELEASE.md b/RELEASE.md index 0323a852..91cdffe9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,6 @@ # Releases -As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][6] and let us know. +As Elastalert 2 is a community-maintained project, releases will typically contain unrelated contributions without a common theme. It's up to the maintainers to determine when the project is ready for a release, however, if you are looking to use a newly merged feature that hasn't yet been released, feel free to open a [discussion][5] and let us know. Maintainers, when creating a new release, follow the procedure below: @@ -14,19 +14,23 @@ Maintainers, when creating a new release, follow the procedure below: - [values.yaml](chart/elastalert2/values.yaml): Match the default image version to the new release version. - [README.md](chart/elastalert2/README.md): Match the default image version to the new release version. 3. Double-check that the Docker image successfully built the latest image. -4. Create a [new][7] release. +4. Create a [new][1] release. - The title (and tag) of the release will be the same value as the new version determined in step 1. - The description of the release will contain a bulleted list of all merged pull requests, in the following format: - `- PR/commit message #000 - @committer` + `- change description #PR - @committer` + Ex: + `- Added new Foobar alerts #12345 - @jertel` - Check the box to 'Create a discussion for this release'. - Save the draft. 5. Verify that artifacts have been published: - - Python PIP package was [published][9] successfully. - - Helm chart has been [published][10] successfully. - - Docker Hub image was [published][8] successfully. + - Python PIP package was [published][2] successfully. + - Helm chart has been [published][3] successfully. + - Docker Hub image was [published][4] successfully. 6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. 7. Publish the release. -[8]: https://hub.docker.com/r/jertel/elastalert2/builds -[9]: https://github.com/jertel/elastalert2/actions/workflows/python-publish.yml -[10]: https://github.com/jertel/elastalert2/actions/workflows/upload_chart.yml \ No newline at end of file +[1]: https://github.com/jertel/elastalert2/releases/new +[2]: https://hub.docker.com/r/jertel/elastalert2/builds +[3]: https://github.com/jertel/elastalert2/actions/workflows/python-publish.yml +[4]: https://github.com/jertel/elastalert2/actions/workflows/upload_chart.yml +[5]: https://github.com/jertel/elastalert2/discussions \ No newline at end of file From d47867d9f687f30399548c974c82715685babf15 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:30:13 -0400 Subject: [PATCH 126/128] Split out release procedure from README into its own file --- RELEASE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 91cdffe9..b54189da 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -23,9 +23,9 @@ Maintainers, when creating a new release, follow the procedure below: - Check the box to 'Create a discussion for this release'. - Save the draft. 5. Verify that artifacts have been published: - - Python PIP package was [published][2] successfully. - - Helm chart has been [published][3] successfully. - - Docker Hub image was [published][4] successfully. + - Python PIP package was [published][3] successfully. + - Helm chart has been [published][4] successfully. + - Docker Hub image was [published][2] successfully. 6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. 7. Publish the release. From 14a4c93a5800be4998205a3c4e10cdfed33b3f92 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 20:34:00 -0400 Subject: [PATCH 127/128] Prepare release 2.0.3 to verify publish actions --- RELEASE.md | 2 +- chart/elastalert2/Chart.yaml | 4 ++-- chart/elastalert2/README.md | 2 +- chart/elastalert2/values.yaml | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index b54189da..fe975661 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -25,7 +25,7 @@ Maintainers, when creating a new release, follow the procedure below: 5. Verify that artifacts have been published: - Python PIP package was [published][3] successfully. - Helm chart has been [published][4] successfully. - - Docker Hub image was [published][2] successfully. + - Docker Hub image was [tagged][2] successfully. 6. Wait a minimum of a few hours for community feedback in case someone notices a problem with the the upcoming release. 7. Publish the release. diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml index 813b4808..4d469c10 100644 --- a/chart/elastalert2/Chart.yaml +++ b/chart/elastalert2/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: Elastalert 2 is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. name: elastalert2 -version: 2.0.2 -appVersion: 2.0.2 +version: 2.0.3 +appVersion: 2.0.3 home: https://github.com/jertel/elastalert2 sources: - https://github.com/jertel/elastalert2 diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index a3dd5c91..86132ff5 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -47,7 +47,7 @@ The command removes all the Kubernetes components associated with the chart and | Parameter | Description | Default | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|---------------------------------| | `image.repository` | docker image | jertel/elastalert-docker | -| `image.tag` | docker image tag | 2.0.2 | +| `image.tag` | docker image tag | 2.0.3 | | `image.pullPolicy` | image pull policy | IfNotPresent | | `podAnnotations` | Annotations to be added to pods | {} | | `command` | command override for container | `NULL` | diff --git a/chart/elastalert2/values.yaml b/chart/elastalert2/values.yaml index c3a24acb..d54f54bd 100644 --- a/chart/elastalert2/values.yaml +++ b/chart/elastalert2/values.yaml @@ -25,7 +25,7 @@ image: # docker image repository: jertel/elastalert2 # docker image tag - tag: 2.0.2 + tag: 2.0.3 pullPolicy: IfNotPresent resources: {} diff --git a/setup.py b/setup.py index 058ca531..eab96832 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert2', - version='2.0.2', + version='2.0.3', description='Runs custom filters on Elasticsearch and alerts on matches', setup_requires='setuptools', license='Apache 2.0', From 92f84f7a8340bedf8d3cb94e27381ac015300aae Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 24 Apr 2021 22:25:25 -0400 Subject: [PATCH 128/128] Use consistent description for Helm Chart and Python PIP library --- chart/elastalert2/Chart.yaml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml index 4d469c10..23f24baf 100644 --- a/chart/elastalert2/Chart.yaml +++ b/chart/elastalert2/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -description: Elastalert 2 is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. +description: Automated rule-based alerting for Elasticsearch name: elastalert2 version: 2.0.3 appVersion: 2.0.3 diff --git a/setup.py b/setup.py index eab96832..6c3f620a 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='elastalert2', version='2.0.3', - description='Runs custom filters on Elasticsearch and alerts on matches', + description='Automated rule-based alerting for Elasticsearch', setup_requires='setuptools', license='Apache 2.0', classifiers=[