From b8fea4e5ed62967cd3cdb65664a95557533476c0 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 30 Jul 2019 12:30:58 -0400 Subject: [PATCH 1/2] Support memory profiling metrics --- .../datadog_checks/base/checks/base.py | 6 ++++- .../datadog_checks/base/utils/agent/memory.py | 25 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 80aa36c27112e..04daf6f3cea37 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -521,9 +521,13 @@ def run(self): from ..utils.agent.memory import TRACE_LOCK, profile_memory with TRACE_LOCK: - profile_memory( + metrics = profile_memory( self.check, self.init_config, namespaces=self.check_id.split(':', 1), args=(instance,) ) + + tags = ['check_name:{}'.format(self.name), 'check_version:{}'.format(self.check_version)] + for m in metrics: + self.gauge(m.name, m.value, tags=tags) else: self.check(instance) diff --git a/datadog_checks_base/datadog_checks/base/utils/agent/memory.py b/datadog_checks_base/datadog_checks/base/utils/agent/memory.py index e7ccafaf50cc1..95c6c85e583cb 100644 --- a/datadog_checks_base/datadog_checks/base/utils/agent/memory.py +++ b/datadog_checks_base/datadog_checks/base/utils/agent/memory.py @@ -33,6 +33,14 @@ TRACE_LOCK = threading.Lock() +class MemoryProfileMetric(object): + __slots__ = ('name', 'value') + + def __init__(self, name, value): + self.name = 'datadog.agent.profile.memory.{}'.format(name) + self.value = float(value) + + def get_sign(n): return '-' if n < 0 else '+' @@ -91,7 +99,7 @@ def get_unit_formatter(unit): return lambda n: format_units(unit, *convert_units(n, to=unit)) -def write_pretty_top(path, snapshot, unit_formatter, key_type, limit, cumulative): +def gather_top(metrics, path, snapshot, unit_formatter, key_type, limit, cumulative): top_stats = snapshot.statistics(key_type, cumulative=cumulative) with open(path, 'w', encoding='utf-8') as f: @@ -121,9 +129,11 @@ def write_pretty_top(path, snapshot, unit_formatter, key_type, limit, cumulative amount, unit = unit_formatter(total) f.write('Total allocated size: {} {}\n'.format(amount, unit)) + metrics.append(MemoryProfileMetric('check_run_total', total)) -def write_pretty_diff( - path, current_snapshot, previous_snapshot, unit_formatter, key_type, limit, cumulative, diff_order + +def gather_diff( + metrics, path, current_snapshot, previous_snapshot, unit_formatter, key_type, limit, cumulative, diff_order ): top_stats = current_snapshot.compare_to(previous_snapshot, key_type=key_type, cumulative=cumulative) @@ -250,13 +260,16 @@ def profile_memory(f, config, namespaces=None, args=(), kwargs=None): unit = config.get('profile_memory_unit', DEFAULT_UNIT) unit_formatter = get_unit_formatter(unit) + # Metrics to send + metrics = [] + # First, write the prettified snapshot snapshot_dir = os.path.join(location, 'snapshots') if not os.path.isdir(snapshot_dir): os.makedirs(snapshot_dir) new_snapshot = os.path.join(snapshot_dir, get_timestamp_filename('snapshot')) - write_pretty_top(new_snapshot, snapshot, unit_formatter, sort_by, limit, combine) + gather_top(metrics, new_snapshot, snapshot, unit_formatter, sort_by, limit, combine) # Then, compute the diff if there was a previous run previous_snapshot_dump = os.path.join(location, 'last-snapshot') @@ -269,7 +282,9 @@ def profile_memory(f, config, namespaces=None, args=(), kwargs=None): # and write it new_diff = os.path.join(diff_dir, get_timestamp_filename('diff')) - write_pretty_diff(new_diff, snapshot, previous_snapshot, unit_formatter, sort_by, limit, combine, diff_order) + gather_diff(metrics, new_diff, snapshot, previous_snapshot, unit_formatter, sort_by, limit, combine, diff_order) # Finally, dump the current snapshot for doing a diff on the next run snapshot.dump(previous_snapshot_dump) + + return metrics From 871e0ecf43c91bffd847ae55b5211b6394ebec84 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 30 Jul 2019 12:56:05 -0400 Subject: [PATCH 2/2] address review --- .../datadog_checks/base/utils/agent/common.py | 4 ++++ .../datadog_checks/base/utils/agent/memory.py | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 datadog_checks_base/datadog_checks/base/utils/agent/common.py diff --git a/datadog_checks_base/datadog_checks/base/utils/agent/common.py b/datadog_checks_base/datadog_checks/base/utils/agent/common.py new file mode 100644 index 0000000000000..013a150a32155 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/agent/common.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2019 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +METRIC_PROFILE_NAMESPACE = 'datadog.agent.profile' diff --git a/datadog_checks_base/datadog_checks/base/utils/agent/memory.py b/datadog_checks_base/datadog_checks/base/utils/agent/memory.py index 95c6c85e583cb..9ddfc2ae7df81 100644 --- a/datadog_checks_base/datadog_checks/base/utils/agent/memory.py +++ b/datadog_checks_base/datadog_checks/base/utils/agent/memory.py @@ -9,6 +9,8 @@ from binary import BinaryUnits, convert_units +from .common import METRIC_PROFILE_NAMESPACE + try: import tracemalloc except ImportError: @@ -37,7 +39,7 @@ class MemoryProfileMetric(object): __slots__ = ('name', 'value') def __init__(self, name, value): - self.name = 'datadog.agent.profile.memory.{}'.format(name) + self.name = '{}.memory.{}'.format(METRIC_PROFILE_NAMESPACE, name) self.value = float(value) @@ -129,7 +131,7 @@ def gather_top(metrics, path, snapshot, unit_formatter, key_type, limit, cumulat amount, unit = unit_formatter(total) f.write('Total allocated size: {} {}\n'.format(amount, unit)) - metrics.append(MemoryProfileMetric('check_run_total', total)) + metrics.append(MemoryProfileMetric('check_run_alloc', total)) def gather_diff(