From f5f12140f3d748ef37d6162dad3ed097d0a00508 Mon Sep 17 00:00:00 2001 From: Xeph Date: Sun, 15 Aug 2021 17:57:33 +0000 Subject: [PATCH 1/4] Add format() syntax support to percentage match rule --- CHANGELOG.md | 1 + docs/source/ruletypes.rst | 3 +-- elastalert/ruletypes.py | 10 ++++++++-- tests/rules_test.py | 6 ++++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0ad736..0d2e7d71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ ## New features - [Kubernetes] Adding Image Pull Secret to Helm Chart - [#370](https://github.com/jertel/elastalert2/pull/370) - @robrankin - Apply percentage_format_string to match_body percentage value; will appear in new percentage_formatted key - [#387](https://github.com/jertel/elastalert2/pull/387) - @iamxeph +- Make percentage_format_string support format() syntax in addition to old %-formatted syntax - [#](https://github.com/jertel/elastalert2/pull/) - @iamxeph - Add metric_format_string optional configuration for Metric Aggregation to format aggregated value - [#](https://github.com/jertel/elastalert2/pull/) - @iamxeph - Add support for Kibana 7.14 for Kibana Discover - [#392](https://github.com/jertel/elastalert2/pull/392) - @nsano-rururu diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4c6c1f4c..a86586e7 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1347,8 +1347,7 @@ evaluated separately against the threshold(s). ``sync_bucket_interval``: See ``sync_bucket_interval`` in Metric Aggregation rule -``percentage_format_string``: An optional format string to apply to the percentage value in the alert match text. This also adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the formatted value for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. -For example, "%.2f" will round it to 2 decimal places. +``percentage_format_string``: An optional format string applies to the aggregated metric value in the alert match text and match_body. This adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both format() and %-formatted syntax works. For example, both "{:.2f}" and "%.2f" will format '96.6666667' to '96.67'. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language ``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0. diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index bf9223ad..886bac27 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1258,7 +1258,7 @@ def __init__(self, *args): def get_match_str(self, match): percentage_format_string = self.rules.get('percentage_format_string', None) message = 'Percentage violation, value: %s (min: %s max : %s) of %s items\n\n' % ( - percentage_format_string % (match['percentage']) if percentage_format_string else match['percentage'], + self.format_string(percentage_format_string, match['percentage']) if percentage_format_string else match['percentage'], self.rules.get('min_percentage'), self.rules.get('max_percentage'), match['denominator'] @@ -1297,7 +1297,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count} percentage_format_string = self.rules.get('percentage_format_string', None) if percentage_format_string is not None: - match['percentage_formatted'] = percentage_format_string % (match_percentage) + match['percentage_formatted'] = self.format_string(percentage_format_string, match_percentage) if query_key is not None: match = expand_string_into_dict(match, self.rules['query_key'], query_key) self.add_match(match) @@ -1308,3 +1308,9 @@ def percentage_violation(self, match_percentage): if 'min_percentage' in self.rules and match_percentage < self.rules['min_percentage']: return True return False + + def format_string(self, format_config, target_value): + if (format_config.startswith('{')): + return format_config.format(target_value) + else: + return format_config % (target_value) diff --git a/tests/rules_test.py b/tests/rules_test.py index 797d28d9..b47bf10f 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1303,6 +1303,12 @@ def test_percentage_match(): assert '76.1589403974' in rule.get_match_str(rule.matches[0]) assert rule.matches[0]['percentage'] == 76.15894039742994 assert 'percentage_formatted' not in rule.matches[0] + rules['percentage_format_string'] = '{:.2f}' + rule = PercentageMatchRule(rules) + rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(76.666666667, 24)) + assert '76.16' in rule.get_match_str(rule.matches[0]) + assert rule.matches[0]['percentage'] == 76.15894039742994 + assert rule.matches[0]['percentage_formatted'] == '76.16' rules['percentage_format_string'] = '%.2f' rule = PercentageMatchRule(rules) rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(76.666666667, 24)) From 912c93688592097577d5ecf24d37c7ab507a6260 Mon Sep 17 00:00:00 2001 From: Xeph Date: Sun, 15 Aug 2021 18:04:43 +0000 Subject: [PATCH 2/4] fix typo --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a86586e7..d4daee0b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1347,7 +1347,7 @@ evaluated separately against the threshold(s). ``sync_bucket_interval``: See ``sync_bucket_interval`` in Metric Aggregation rule -``percentage_format_string``: An optional format string applies to the aggregated metric value in the alert match text and match_body. This adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both format() and %-formatted syntax works. For example, both "{:.2f}" and "%.2f" will format '96.6666667' to '96.67'. +``percentage_format_string``: An optional format string applies to the percentage value in the alert match text and match_body. This adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both format() and %-formatted syntax works. For example, both "{:.2f}" and "%.2f" will format '96.6666667' to '96.67'. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language ``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0. From 10f838b86687cc2643a3a73b06b669cc59a29e35 Mon Sep 17 00:00:00 2001 From: Xeph Date: Sun, 15 Aug 2021 18:12:29 +0000 Subject: [PATCH 3/4] Add PR number to CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d2e7d71..f099b439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ ## New features - [Kubernetes] Adding Image Pull Secret to Helm Chart - [#370](https://github.com/jertel/elastalert2/pull/370) - @robrankin - Apply percentage_format_string to match_body percentage value; will appear in new percentage_formatted key - [#387](https://github.com/jertel/elastalert2/pull/387) - @iamxeph -- Make percentage_format_string support format() syntax in addition to old %-formatted syntax - [#](https://github.com/jertel/elastalert2/pull/) - @iamxeph +- Make percentage_format_string support format() syntax in addition to old %-formatted syntax - [#403](https://github.com/jertel/elastalert2/pull/403) - @iamxeph - Add metric_format_string optional configuration for Metric Aggregation to format aggregated value - [#](https://github.com/jertel/elastalert2/pull/) - @iamxeph - Add support for Kibana 7.14 for Kibana Discover - [#392](https://github.com/jertel/elastalert2/pull/392) - @nsano-rururu From 352754ad1f1c76e26e51d487b8377f4c0a809da9 Mon Sep 17 00:00:00 2001 From: Xeph Date: Mon, 16 Aug 2021 03:32:15 +0000 Subject: [PATCH 4/4] Move format_string to util class and revise README terms regarding formats --- docs/source/ruletypes.rst | 4 ++-- elastalert/ruletypes.py | 24 ++++++------------------ elastalert/util.py | 15 +++++++++++++++ tests/util_test.py | 9 +++++++++ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d4daee0b..960aa20c 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1263,7 +1263,7 @@ allign with the time elastalert runs, (This both avoid calculations on partial d See: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html#_offset for a more comprehensive explaination. -``metric_format_string``: An optional format string applies to the aggregated metric value in the alert match text and match_body. This adds 'metric_{metric_agg_key}_formatted' value to the match_body in addition to raw, unformatted 'metric_{metric_agg_key}' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both format() and %-formatted syntax works. For example, "{:.2%}" will format '0.966666667' to '96.67%', and "%.2f" will format '0.966666667' to '0.97'. +``metric_format_string``: An optional format string applies to the aggregated metric value in the alert match text and match_body. This adds 'metric_{metric_agg_key}_formatted' value to the match_body in addition to raw, unformatted 'metric_{metric_agg_key}' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both str.format() and %-format syntax works. For example, "{:.2%}" will format '0.966666667' to '96.67%', and "%.2f" will format '0.966666667' to '0.97'. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language @@ -1347,7 +1347,7 @@ evaluated separately against the threshold(s). ``sync_bucket_interval``: See ``sync_bucket_interval`` in Metric Aggregation rule -``percentage_format_string``: An optional format string applies to the percentage value in the alert match text and match_body. This adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both format() and %-formatted syntax works. For example, both "{:.2f}" and "%.2f" will format '96.6666667' to '96.67'. +``percentage_format_string``: An optional format string applies to the percentage value in the alert match text and match_body. This adds 'percentage_formatted' value to the match_body in addition to raw, unformatted 'percentage' value so that you can use the values for ``alert_subject_args`` and ``alert_text_args``. Must be a valid python format string. Both str.format() and %-format syntax works. For example, both "{:.2f}" and "%.2f" will format '96.6666667' to '96.67'. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language ``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0. diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 886bac27..e08c3dc6 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -7,7 +7,7 @@ from elastalert.util import (add_raw_postfix, dt_to_ts, EAException, elastalert_logger, elasticsearch_client, format_index, hashable, lookup_es_key, new_get_event_ts, pretty_ts, total_seconds, - ts_now, ts_to_dt, expand_string_into_dict) + ts_now, ts_to_dt, expand_string_into_dict, format_string) class RuleType(object): @@ -1072,7 +1072,7 @@ def get_match_str(self, match): message = 'Threshold violation, %s:%s %s (min: %s max : %s) \n\n' % ( self.rules['metric_agg_type'], self.rules['metric_agg_key'], - self.format_string(metric_format_string, match[self.metric_key]) if metric_format_string else match[self.metric_key], + format_string(metric_format_string, match[self.metric_key]) if metric_format_string else match[self.metric_key], self.rules.get('min_threshold'), self.rules.get('max_threshold') ) @@ -1098,7 +1098,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): self.metric_key: metric_val} metric_format_string = self.rules.get('metric_format_string', None) if metric_format_string is not None: - match[self.metric_key +'_formatted'] = self.format_string(metric_format_string, metric_val) + match[self.metric_key +'_formatted'] = format_string(metric_format_string, metric_val) if query_key is not None: match = expand_string_into_dict(match, self.rules['query_key'], query_key) self.add_match(match) @@ -1140,12 +1140,6 @@ def crossed_thresholds(self, metric_value): return True return False - def format_string(self, format_config, target_value): - if (format_config.startswith('{')): - return format_config.format(target_value) - else: - return format_config % (target_value) - class SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule): """ A rule that matches when there is a spike in an aggregated event compared to its reference point """ @@ -1258,7 +1252,7 @@ def __init__(self, *args): def get_match_str(self, match): percentage_format_string = self.rules.get('percentage_format_string', None) message = 'Percentage violation, value: %s (min: %s max : %s) of %s items\n\n' % ( - self.format_string(percentage_format_string, match['percentage']) if percentage_format_string else match['percentage'], + format_string(percentage_format_string, match['percentage']) if percentage_format_string else match['percentage'], self.rules.get('min_percentage'), self.rules.get('max_percentage'), match['denominator'] @@ -1297,7 +1291,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count} percentage_format_string = self.rules.get('percentage_format_string', None) if percentage_format_string is not None: - match['percentage_formatted'] = self.format_string(percentage_format_string, match_percentage) + match['percentage_formatted'] = format_string(percentage_format_string, match_percentage) if query_key is not None: match = expand_string_into_dict(match, self.rules['query_key'], query_key) self.add_match(match) @@ -1307,10 +1301,4 @@ def percentage_violation(self, match_percentage): return True if 'min_percentage' in self.rules and match_percentage < self.rules['min_percentage']: return True - return False - - def format_string(self, format_config, target_value): - if (format_config.startswith('{')): - return format_config.format(target_value) - else: - return format_config % (target_value) + return False \ No newline at end of file diff --git a/elastalert/util.py b/elastalert/util.py index 82d84888..ac59a85e 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -518,3 +518,18 @@ def expand_string_into_dict(dictionary, string , value, sep='.'): field1, new_string = string.split(sep, 1) dictionary[field1] = _expand_string_into_dict(new_string, value) return dictionary + + +def format_string(format_config, target_value): + """ + Formats number, supporting %-format and str.format() syntax. + + :param format_config: string format syntax, for example '{:.2%}' or '%.2f' + :param target_value: number to format + :rtype: string + """ + if (format_config.startswith('{')): + return format_config.format(target_value) + else: + return format_config % (target_value) + diff --git a/tests/util_test.py b/tests/util_test.py index 89090de2..704164c1 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -35,6 +35,7 @@ from elastalert.util import ts_utc_to_tz from elastalert.util import expand_string_into_dict from elastalert.util import unixms_to_dt +from elastalert.util import format_string @pytest.mark.parametrize('spec, expected_delta', [ @@ -502,3 +503,11 @@ def test_dt_to_int(): actual = dt_to_int(dt) expected = 1625529600000 assert expected == actual + + +def test_format_string(): + target = 0.966666667 + expected_percent_formatting = '0.97' + assert format_string('%.2f', target) == expected_percent_formatting + expected_str_formatting = '96.67%' + assert format_string('{:.2%}', target) == expected_str_formatting