Skip to content

Commit

Permalink
[OpenMetricsV2] Add an option to send sum and count information when …
Browse files Browse the repository at this point in the history 
…using distribution metrics (#9301)

* [OpenMetricsV2] Send sum and count information when using distribution metrics

* put behind flag

* document option
  • Loading branch information
@ofek
ofek authored May 11, 2021
1 parent 90532d9 commit 54232b6
Show file tree
Hide file tree
Showing 5 changed files with 225 additions and 17 deletions.
6 changes: 5 additions & 1 deletion datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ def __init__(self, check, config):
self.check = check
self.logger = check.log
self.cache_metric_wildcards = is_affirmative(config.get('cache_metric_wildcards', True))
self.histogram_buckets_as_distributions = is_affirmative(
self.collect_counters_with_distributions = is_affirmative(
config.get('collect_counters_with_distributions', False)
)
self.histogram_buckets_as_distributions = self.collect_counters_with_distributions or is_affirmative(
config.get('histogram_buckets_as_distributions', False)
)
self.collect_histogram_buckets = self.histogram_buckets_as_distributions or is_affirmative(
Expand All @@ -29,6 +32,7 @@ def __init__(self, check, config):

# Accessible to every transformer
self.global_options = {
'collect_counters_with_distributions': self.collect_counters_with_distributions,
'collect_histogram_buckets': self.collect_histogram_buckets,
'histogram_buckets_as_distributions': self.histogram_buckets_as_distributions,
'non_cumulative_histogram_buckets': self.non_cumulative_histogram_buckets,
Expand Down
75 changes: 59 additions & 16 deletions datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/transformers/histogram.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,27 +12,70 @@ def get_histogram(check, metric_name, modifiers, global_options):
"""
if global_options['collect_histogram_buckets']:
if global_options['histogram_buckets_as_distributions']:
submit_histogram_bucket_method = check.submit_histogram_bucket
logger = check.log
submit_histogram_bucket_method = check.submit_histogram_bucket

def histogram(metric, sample_data, runtime_data):
for sample, tags, hostname in decumulate_histogram_buckets(sample_data):
if not sample.name.endswith('_bucket'):
continue
if global_options['collect_counters_with_distributions']:
monotonic_count_method = check.monotonic_count
sum_metric = f'{metric_name}.sum'
count_metric = f'{metric_name}.count'

lower_bound = canonicalize_numeric_label(sample.labels['lower_bound'])
upper_bound = canonicalize_numeric_label(sample.labels['upper_bound'])
def histogram(metric, sample_data, runtime_data):
has_successfully_executed = runtime_data['has_successfully_executed']

if lower_bound == upper_bound:
# this can happen for -inf/-inf bucket that we don't want to send (always 0)
logger.warning(
'Metric: %s has bucket boundaries equal, skipping: %s', metric_name, sample.labels
)
continue
for sample, tags, hostname in decumulate_histogram_buckets(sample_data):
sample_name = sample.name
if sample_name.endswith('_sum'):
monotonic_count_method(
sum_metric,
sample.value,
tags=tags,
hostname=hostname,
flush_first_value=has_successfully_executed,
)
elif sample_name.endswith('_count'):
monotonic_count_method(
count_metric,
sample.value,
tags=tags,
hostname=hostname,
flush_first_value=has_successfully_executed,
)
elif sample_name.endswith('_bucket'):
lower_bound = canonicalize_numeric_label(sample.labels['lower_bound'])
upper_bound = canonicalize_numeric_label(sample.labels['upper_bound'])

if lower_bound == upper_bound:
# this can happen for -inf/-inf bucket that we don't want to send (always 0)
logger.warning(
'Metric: %s has bucket boundaries equal, skipping: %s', metric_name, sample.labels
)
continue

submit_histogram_bucket_method(
metric_name, sample.value, lower_bound, upper_bound, True, hostname, tags
)

submit_histogram_bucket_method(
metric_name, sample.value, lower_bound, upper_bound, True, hostname, tags
)
else:

def histogram(metric, sample_data, runtime_data):
for sample, tags, hostname in decumulate_histogram_buckets(sample_data):
if not sample.name.endswith('_bucket'):
continue

lower_bound = canonicalize_numeric_label(sample.labels['lower_bound'])
upper_bound = canonicalize_numeric_label(sample.labels['upper_bound'])

if lower_bound == upper_bound:
# this can happen for -inf/-inf bucket that we don't want to send (always 0)
logger.warning(
'Metric: %s has bucket boundaries equal, skipping: %s', metric_name, sample.labels
)
continue

submit_histogram_bucket_method(
metric_name, sample.value, lower_bound, upper_bound, True, hostname, tags
)

else:
monotonic_count_method = check.monotonic_count
Expand Down
146 changes: 146 additions & 0 deletions datadog_checks_base/tests/openmetrics/test_transformers/test_histogram.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,3 +453,149 @@ def test_histogram_buckets_as_distributions(aggregator, dd_run_check, mock_http_
)

aggregator.assert_all_metrics_covered()


def test_histogram_buckets_as_distributions_with_counters(aggregator, dd_run_check, mock_http_response):
payload = """
# HELP rest_client_request_latency_seconds Request latency in seconds. Broken down by verb and URL.
# TYPE rest_client_request_latency_seconds histogram
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.004"} 702
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.001"} 254
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.002"} 621
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.008"} 727
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.016"} 738
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.032"} 744
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.064"} 748
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.128"} 754
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.256"} 755
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="0.512"} 755
rest_client_request_latency_seconds_bucket{url="http://127.0.0.1:8080/api",verb="GET",le="+Inf"} 755
rest_client_request_latency_seconds_sum{url="http://127.0.0.1:8080/api",verb="GET"} 2.185820220000001
rest_client_request_latency_seconds_count{url="http://127.0.0.1:8080/api",verb="GET"} 755
"""
mock_http_response(payload)
check = get_check(
{
'metrics': ['.+'],
'collect_counters_with_distributions': True,
# Implicitly activated
'histogram_buckets_as_distributions': False,
'collect_histogram_buckets': False,
}
)
dd_run_check(check)

aggregator.assert_metric(
'test.rest_client_request_latency_seconds.sum',
2.185820220000001,
metric_type=aggregator.MONOTONIC_COUNT,
tags=['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET'],
)
aggregator.assert_metric(
'test.rest_client_request_latency_seconds.count',
755,
metric_type=aggregator.MONOTONIC_COUNT,
tags=['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET'],
)

aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
81,
0.002,
0.004,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.004', 'lower_bound:0.002'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
254,
0,
0.001,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.001', 'lower_bound:0'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
367,
0.001,
0.002,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.002', 'lower_bound:0.001'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
25,
0.004,
0.008,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.008', 'lower_bound:0.004'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
11,
0.008,
0.016,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.016', 'lower_bound:0.008'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
6,
0.016,
0.032,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.032', 'lower_bound:0.016'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
4,
0.032,
0.064,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.064', 'lower_bound:0.032'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
6,
0.064,
0.128,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.128', 'lower_bound:0.064'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
1,
0.128,
0.256,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.256', 'lower_bound:0.128'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
0,
0.256,
0.512,
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:0.512', 'lower_bound:0.256'],
)
aggregator.assert_histogram_bucket(
'test.rest_client_request_latency_seconds',
0,
0.512,
float('Inf'),
True,
check.hostname,
['endpoint:test', 'url:http://127.0.0.1:8080/api', 'verb:GET', 'upper_bound:inf', 'lower_bound:0.512'],
)

aggregator.assert_all_metrics_covered()
8 changes: 8 additions & 0 deletions ..._checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,14 @@
value:
example: false
type: boolean
- name: collect_counters_with_distributions
description: |
Whether or not to also collect the observation counter metrics ending in `.sum` and `.count`
when sending histogram buckets as Datadog distribution metrics. This implicitly enables the
`histogram_buckets_as_distributions` option.
value:
example: false
type: boolean
- name: share_labels
description: |
This mapping allows for the sharing of labels across multiple metrics. The keys represent the
Expand Down
7 changes: 7 additions & 0 deletions kong/datadog_checks/kong/data/conf.yaml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,13 @@ instances:
#
# histogram_buckets_as_distributions: false

## @param collect_counters_with_distributions - boolean - optional - default: false
## Whether or not to also collect the observation counter metrics ending in `.sum` and `.count`
## when sending histogram buckets as Datadog distribution metrics. This implicitly enables the
## `histogram_buckets_as_distributions` option.
#
# collect_counters_with_distributions: false

## @param share_labels - mapping - optional
## This mapping allows for the sharing of labels across multiple metrics. The keys represent the
## exposed metrics from which to share labels, and the values are mappings that configure the
Expand Down

0 comments on commit 54232b6

Please sign in to comment.