From 11c9f6fe5bf1dfd2cf24e096317b2c08df3caf03 Mon Sep 17 00:00:00 2001 From: dequis Date: Mon, 15 Nov 2021 20:35:44 +0100 Subject: [PATCH 1/3] Add metric_agg_script to MetricAggregationRule Copied from SpikeMetricAggregationRule, which originally copied generate_aggregation_query from MetricAggregationRule and added these two lines. So this commit just makes the two generate_aggregation_query look the same, which just works and I'm not really sure why no one has done it before. --- docs/source/ruletypes.rst | 8 +++++++- elastalert/ruletypes.py | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4d62c506..5caf8b07 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1318,7 +1318,7 @@ default this is ``buffer_time``. This rule requires: ``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be -supported by the specified aggregation type. +supported by the specified aggregation type. If using a scripted field via ``metric_agg_script``, this is the name for your scripted field ``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg', 'sum', 'cardinality', 'value_count'. @@ -1336,6 +1336,12 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the threshold(s). +``metric_agg_script``: A `Painless` formatted script describing how to calculate your metric on-the-fly:: + + metric_agg_key: myScriptedMetric + metric_agg_script: + script: doc['field1'].value * doc['field2'].value + ``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 5c811156..3d99efe1 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1088,6 +1088,8 @@ def get_match_str(self, match): return message def generate_aggregation_query(self): + if self.rules.get('metric_agg_script'): + return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} query = {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} if self.rules['metric_agg_type'] in self.allowed_percent_aggregations: query[self.metric_key][self.rules['metric_agg_type']]['percents'] = [self.rules['percentile_range']] @@ -1175,7 +1177,7 @@ def __init__(self, *args): self.rules['aggregation_query_element'] = self.generate_aggregation_query() def generate_aggregation_query(self): - """Lifted from MetricAggregationRule, added support for scripted fields""" + """Lifted from MetricAggregationRule""" if self.rules.get('metric_agg_script'): return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} query = {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} From 71aaa258767ccabad60224588ef81cbb7e8cbb25 Mon Sep 17 00:00:00 2001 From: dequis Date: Mon, 15 Nov 2021 20:55:06 +0100 Subject: [PATCH 2/3] Add changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc9233d0..91aef474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ - None ## New features -- None +- Add metric_agg_script to MetricAggregationRule [#558](https://github.com/jertel/elastalert2/pull/558) - @dequis ## Other changes - sphinx 4.2.0 to 4.3.0 and tzlocal==2.1 - [#561](https://github.com/jertel/elastalert2/pull/561) - @nsano-rururu From e7f0575a746de8c8b51e4ee7a4ee960b63d7a7e3 Mon Sep 17 00:00:00 2001 From: dequis Date: Mon, 22 Nov 2021 15:30:00 +0100 Subject: [PATCH 3/3] Add test_metric_aggregation_scripted --- tests/rules_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/rules_test.py b/tests/rules_test.py index 83d1c18a..e440dabe 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1261,6 +1261,22 @@ def test_metric_aggregation_complex_query_key_bucket_interval(): assert rule.matches[1]['sub_qk'] == 'sub_qk_val1' +def test_metric_aggregation_scripted(): + script_body = "doc['some_threshold'].value - doc['cpu_pct'].value" + rules = {'buffer_time': datetime.timedelta(minutes=5), + 'timestamp_field': '@timestamp', + 'metric_agg_type': 'avg', + 'metric_agg_key': 'cpu_pct', + 'metric_agg_script': {"script": script_body}, + 'min_threshold': 0.0} + + rule = MetricAggregationRule(rules) + assert rule.rules['aggregation_query_element'] == {'metric_cpu_pct_avg': {'avg': {'script': script_body}}} + + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': -0.5}}) + assert rule.matches[0]['metric_cpu_pct_avg'] == -0.5 + + def test_percentage_match(): rules = {'match_bucket_filter': {'term': 'term_val'}, 'buffer_time': datetime.timedelta(minutes=5),