Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add native metrics #13474

Merged
merged 7 commits into from
Dec 9, 2022
Merged

Add native metrics #13474

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
dd2c289
Add native metrics
yzhan289 Dec 7, 2022
b3ca600
Fix e2e test
yzhan289 Dec 7, 2022
c9765d8
Move metadata.csv change to separate branch
yzhan289 Dec 7, 2022
266ad98
Renamed some methods
jose-manuel-almaza Dec 9, 2022
1923f92
Fixed problem merging metrics
jose-manuel-almaza Dec 9, 2022
5c641c8
Fixed unit tests
jose-manuel-almaza Dec 9, 2022
6555977
Merge remote-tracking branch 'origin/az/cloudera' into az/cloudera-na…
jose-manuel-almaza Dec 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cloudera/datadog_checks/cloudera/api_client_v7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from datadog_checks.base import AgentCheck
from datadog_checks.cloudera.api_client import ApiClient
from datadog_checks.cloudera.entity_status import ENTITY_STATUS
from datadog_checks.cloudera.metrics import TIMESERIES_METRICS
from datadog_checks.cloudera.metrics import NATIVE_METRICS, TIMESERIES_METRICS

from .common import CLUSTER_HEALTH, HOST_HEALTH

Expand Down Expand Up @@ -59,6 +59,7 @@ def _collect_hosts(self, cluster_name):
tags = self._collect_host_tags(host)

if host.host_id:
self._collect_host_native_metrics(host, tags)
self._collect_host_metrics(host, tags)
self._collect_role_metrics(host, tags)
self._collect_disk_metrics(host, tags)
Expand All @@ -81,6 +82,10 @@ def _collect_host_service_check(self, host, tags):
self._log.debug('host_entity_status: %s', host_entity_status)
self._check.service_check(HOST_HEALTH, host_entity_status, tags=tags)

def _collect_host_native_metrics(self, host, tags):
for metric in NATIVE_METRICS['host']:
self._check.gauge(f"host.{metric}", getattr(host, metric), tags)

def _collect_host_metrics(self, host, tags):
metric_names = ','.join(f'last({metric}) AS {metric}' for metric in TIMESERIES_METRICS['host'])
query = f'SELECT {metric_names} WHERE hostId="{host.host_id}" AND category=HOST'
Expand Down
19 changes: 17 additions & 2 deletions cloudera/datadog_checks/cloudera/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@
],
}

NATIVE_METRICS = {}
NATIVE_METRICS = {
'host': [
'num_cores',
'num_physical_cores',
'total_phys_mem_bytes',
]
}


def merge_dicts(dict_1, dict_2):
dict_3 = {**dict_1, **dict_2}
for key, values in dict_1.items():
for value in values:
dict_3[key].append(value)
return dict_3


METRICS = {**NATIVE_METRICS, **TIMESERIES_METRICS}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To reviewer: using {**dict_1, **dict_2} will combine two dictionaries, although it will give priority to dict_2 if there are matching keys.

As a result, METRICS did not contain num_cores, num_physical_cores, and total_phys_mem_bytes since TIMESERIES_METRICS's host key is prioritized over NATIVE_METRICS's host key.

METRICS = merge_dicts(NATIVE_METRICS, TIMESERIES_METRICS)
44 changes: 44 additions & 0 deletions cloudera/metadata.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,46 @@
metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric
cloudera.cluster.cpu_percent_across_hosts,gauge,,percent,,CPU usage in percent across hosts for this cluster,0,cloudera,cluster cpu percent across hosts,
cloudera.cluster.total_bytes_receive_rate_across_network_interfaces,gauge,,,,,0,cloudera,,
cloudera.cluster.total_bytes_transmit_rate_across_network_interfaces,gauge,,,,,0,cloudera,,
cloudera.cluster.total_read_bytes_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.cluster.total_write_bytes_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.disk.await_read_time,gauge,,,,,0,cloudera,,
cloudera.disk.await_time,gauge,,,,,0,cloudera,,
cloudera.disk.await_write_time,gauge,,,,,0,cloudera,,
cloudera.disk.service_time,gauge,,,,,0,cloudera,,
cloudera.host.alerts_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_iowait_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_irq_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_nice_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_soft_irq_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_steal_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_system_rate,gauge,,,,,0,cloudera,,
cloudera.host.cpu_user_rate,gauge,,,,,0,cloudera,,
cloudera.host.events_critical_rate,gauge,,,,,0,cloudera,,
cloudera.host.events_important_rate,gauge,,,,,0,cloudera,,
cloudera.host.health_bad_rate,gauge,,,,,0,cloudera,,
cloudera.host.health_concerning_rate,gauge,,,,,0,cloudera,,
cloudera.host.health_disabled_rate,gauge,,,,,0,cloudera,,
cloudera.host.health_good_rate,gauge,,,,,0,cloudera,,
cloudera.host.health_unknown_rate,gauge,,,,,0,cloudera,,
cloudera.host.load_15,gauge,,,,,0,cloudera,,
cloudera.host.load_1,gauge,,,,,0,cloudera,,
cloudera.host.load_5,gauge,,,,,0,cloudera,,
cloudera.host.num_cores,gauge,,,,,0,cloudera,,
cloudera.host.num_physical_cores,gauge,,,,,0,cloudera,,
cloudera.host.physical_memory_buffers,gauge,,,,,0,cloudera,,
cloudera.host.physical_memory_cached,gauge,,,,,0,cloudera,,
cloudera.host.physical_memory_total,gauge,,,,,0,cloudera,,
cloudera.host.physical_memory_used,gauge,,,,,0,cloudera,,
cloudera.host.swap_out_rate,gauge,,,,,0,cloudera,,
cloudera.host.swap_used,gauge,,,,,0,cloudera,,
cloudera.host.total_bytes_receive_rate_across_network_interfaces,gauge,,,,,0,cloudera,,
cloudera.host.total_bytes_transmit_rate_across_network_interfaces,gauge,,,,,0,cloudera,,
cloudera.host.total_phys_mem_bytes,gauge,,,,,0,cloudera,,
cloudera.host.total_read_bytes_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.host.total_read_ios_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.host.total_write_bytes_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.host.total_write_ios_rate_across_disks,gauge,,,,,0,cloudera,,
cloudera.role.cpu_system_rate,gauge,,,,,0,cloudera,,
cloudera.role.cpu_user_rate,gauge,,,,,0,cloudera,,
cloudera.role.mem_rss,gauge,,,,,0,cloudera,,
3 changes: 3 additions & 0 deletions cloudera/tests/test_e2e.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from datadog_checks.cloudera import ClouderaCheck
from datadog_checks.cloudera.metrics import METRICS
from datadog_checks.dev.utils import get_metadata_metrics


@pytest.mark.e2e
Expand All @@ -17,3 +18,5 @@ def test_e2e(dd_agent_check, instance):
# caddy test env is supposed to be in BAD_HEALTH
aggregator.assert_service_check('cloudera.cluster.health', ClouderaCheck.CRITICAL, message="BAD_HEALTH")
aggregator.assert_service_check('cloudera.host.health', ClouderaCheck.OK)
aggregator.assert_all_metrics_covered()
aggregator.assert_metrics_using_metadata(get_metadata_metrics())
3 changes: 3 additions & 0 deletions cloudera/tests/test_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from datadog_checks.cloudera import ClouderaCheck
from datadog_checks.cloudera.metrics import METRICS
from datadog_checks.dev.utils import get_metadata_metrics


@pytest.mark.usefixtures('dd_environment')
Expand Down Expand Up @@ -41,3 +42,5 @@ def test_given_api_v48_endpoint_when_check_runs_then_service_check_ok_and_metric
# caddy test env is supposed to be in BAD_HEALTH
aggregator.assert_service_check('cloudera.cluster.health', ClouderaCheck.CRITICAL, message="BAD_HEALTH")
aggregator.assert_service_check('cloudera.host.health', ClouderaCheck.OK)
aggregator.assert_all_metrics_covered()
aggregator.assert_metrics_using_metadata(get_metadata_metrics())