Add User Profiles support (#14752)

snmp/datadog_checks/snmp/data/default_profiles/README.md

@@ -0,0 +1 @@
+The `default_profiles` folder is used to store datadog default profiles.

snmp/datadog_checks/snmp/data/profiles/README.md

@@ -0,0 +1 @@
+The `profiles` folder is used to store user custom profiles.

snmp/datadog_checks/snmp/utils.py

@@ -48,25 +48,36 @@ def get_profile_definition(profile):
     return profile['definition']
 
 
-def _get_profiles_confd_root():
+def _get_profiles_confd_user_root():
     # type: () -> str
     # NOTE: this separate helper function exists for mocking purposes.
     confd = get_config('confd_path')
     return os.path.join(confd, 'snmp.d', 'profiles')
 
 
+def _get_profiles_confd_default_root():
+    # type: () -> str
+    # NOTE: this separate helper function exists for mocking purposes.
+    confd = get_config('confd_path')
+    return os.path.join(confd, 'snmp.d', 'default_profiles')
+
+
 def _get_profiles_site_root():
     # type: () -> str
     here = os.path.dirname(__file__)
-    return os.path.join(here, 'data', 'profiles')
+    return os.path.join(here, 'data', 'default_profiles')
 
 
 def _resolve_definition_file(definition_file):
     # type: (str) -> str
     if os.path.isabs(definition_file):
         return definition_file
 
-    definition_conf_file = os.path.join(_get_profiles_confd_root(), definition_file)
+    definition_conf_file = os.path.join(_get_profiles_confd_user_root(), definition_file)
+    if os.path.isfile(definition_conf_file):
+        return definition_conf_file
+
+    definition_conf_file = os.path.join(_get_profiles_confd_default_root(), definition_file)
     if os.path.isfile(definition_conf_file):
         return definition_conf_file
 
@@ -107,7 +118,9 @@ def recursively_expand_base_profiles(definition):
 
 def _iter_default_profile_file_paths():
     # type: () -> Iterator[str]
-    paths = [_get_profiles_site_root(), _get_profiles_confd_root()]
+
+    # the order of the path is important, the first profile with specific name found will take precedence
+    paths = [_get_profiles_confd_user_root(), _get_profiles_confd_default_root(), _get_profiles_site_root()]
 
     for path in paths:
         if not os.path.isdir(path):
@@ -138,6 +151,8 @@ def _load_default_profiles():
 
     for path in _iter_default_profile_file_paths():
         name = _get_profile_name(path)
+        if name in profiles:
+            continue
 
         if _is_abstract_profile(name):
             continue

snmp/tests/conftest.py

@@ -17,6 +17,7 @@
 from .common import (
     ACTIVE_ENV_NAME,
     COMPOSE_DIR,
+    HERE,
     PORT,
     SNMP_CONTAINER_NAME,
     SNMP_LISTENER_ENV,
@@ -30,7 +31,11 @@
 E2E_METADATA = {
     'start_commands': [
         # Ensure the Agent has access to profile definition files and auto_conf.
-        'cp -r /home/snmp/datadog_checks/snmp/data/profiles /etc/datadog-agent/conf.d/snmp.d/',
+        'cp -r /home/snmp/datadog_checks/snmp/data/default_profiles /etc/datadog-agent/conf.d/snmp.d/',
+    ],
+    'docker_volumes': [
+        # Mount mock user profiles
+        '{}/fixtures/user_profiles:/etc/datadog-agent/conf.d/snmp.d/profiles'.format(HERE),
     ],
 }
 
@@ -53,9 +58,9 @@ def dd_environment():
         with docker_run(os.path.join(COMPOSE_DIR, 'docker-compose.yaml'), env_vars=env, log_patterns="Listening at"):
             if SNMP_LISTENER_ENV == 'true':
                 instance_config = {}
-                new_e2e_metadata['docker_volumes'] = [
-                    '{}:/etc/datadog-agent/datadog.yaml'.format(create_datadog_conf_file(tmp_dir))
-                ]
+                new_e2e_metadata['docker_volumes'].append(
+                    '{}:/etc/datadog-agent/datadog.yaml'.format(create_datadog_conf_file(tmp_dir)),
+                )
             else:
                 instance_config = generate_container_instance_config([])
                 instance_config['init_config'].update(

snmp/tests/fixtures/user_profiles/apc_ups_user.yaml

@@ -0,0 +1,17 @@
+extends:
+  - apc_ups.yaml
+
+sysobjectid: 1.3.6.1.4.1.318.1.999
+
+metadata:
+  device:
+    fields:
+      vendor:
+        value: "apc"
+      serial_number:
+        value: "fake-user-serial-num"
+metrics:
+  - MIB: PowerNet-MIB
+    symbol:
+      OID: 1.3.6.1.4.1.318.1.1.1.2.2.5.0
+      name: upsAdvBatteryNumOfBattPacks_userMetric

snmp/tests/test_e2e_core.py

@@ -7,6 +7,7 @@
 from tests.common import SNMP_CONTAINER_NAME
 
 from . import common, metrics
+from .test_e2e_core_metadata import assert_device_metadata
 
 pytestmark = [pytest.mark.e2e, common.py3_plus_only, common.snmp_integration_only]
 
@@ -66,6 +67,67 @@ def test_e2e_v1_with_apc_ups_profile_batch_size_1(dd_agent_check):
     assert_apc_ups_metrics(dd_agent_check, config)
 
 
+def test_e2e_user_profiles(dd_agent_check):
+    config = common.generate_container_instance_config([])
+    instance = config['instances'][0]
+    instance.update(
+        {
+            'loader': 'core',
+            'community_string': 'apc_ups_user',
+        }
+    )
+    device_ip = instance['ip_address']
+
+    aggregator = common.dd_agent_check_wrapper(dd_agent_check, config, rate=True)
+    profile_tags = [
+        'snmp_profile:apc_ups_user',
+        'model:APC Smart-UPS 600',
+        'firmware_version:2.0.3-test',
+        'serial_num:test_serial',
+        'ups_name:testIdentName',
+        'device_namespace:default',
+    ]
+    tags = profile_tags + ["snmp_device:{}".format(device_ip)]
+
+    aggregator.assert_metric('snmp.upsAdvBatteryNumOfBattPacks', metric_type=aggregator.GAUGE, tags=tags, count=2)
+    aggregator.assert_metric(
+        'snmp.upsAdvBatteryNumOfBattPacks_userMetric', metric_type=aggregator.GAUGE, tags=tags, count=2
+    )
+
+    device = {
+        'description': 'APC Web/SNMP Management Card (MB:v3.9.2 PF:v3.9.2 '
+        'PN:apc_hw02_aos_392.bin AF1:v3.7.2 AN1:apc_hw02_sumx_372.bin '
+        'MN:AP9619 HR:A10 SN: 5A1827E00000 MD:12/04/2007) (Embedded '
+        'PowerNet SNMP Agent SW v2.2 compatible)',
+        'id': 'default:' + device_ip,
+        'id_tags': [
+            'device_namespace:default',
+            'snmp_device:' + device_ip,
+        ],
+        'ip_address': device_ip,
+        'model': 'AP9619',
+        'os_name': 'AOS',
+        'os_version': 'v3.9.2',
+        'product_name': 'APC Smart-UPS 600',
+        'profile': 'apc_ups_user',
+        'serial_number': 'fake-user-serial-num',
+        'status': 1,
+        'sys_object_id': '1.3.6.1.4.1.318.1.999',
+        'tags': [
+            'device_namespace:default',
+            'firmware_version:2.0.3-test',
+            'model:APC Smart-UPS 600',
+            'serial_num:test_serial',
+            'snmp_device:' + device_ip,
+            'snmp_profile:apc_ups_user',
+            'ups_name:testIdentName',
+        ],
+        'vendor': 'apc',
+        'version': '2.0.3-test',
+    }
+    assert_device_metadata(aggregator, device)
+
+
 def assert_apc_ups_metrics(dd_agent_check, config):
     config['init_config']['loader'] = 'core'
     instance = config['instances'][0]
@@ -348,6 +410,9 @@ def test_e2e_core_detect_metrics_using_apc_ups_metrics(dd_agent_check):
 
     for metric in metrics.APC_UPS_METRICS:
         aggregator.assert_metric('snmp.{}'.format(metric), metric_type=aggregator.GAUGE, tags=tags, count=2)
+    aggregator.assert_metric(
+        'snmp.upsAdvBatteryNumOfBattPacks_userMetric', metric_type=aggregator.GAUGE, tags=tags, count=2
+    )
     aggregator.assert_metric(
         'snmp.upsOutletGroupStatusGroupState',
         metric_type=aggregator.GAUGE,

snmp/tests/test_unit.py

@@ -29,7 +29,7 @@
 )
 
 from . import common
-from .utils import mock_profiles_confd_root
+from .utils import mkdir_p, mock_profiles_confd_default_root, mock_profiles_confd_user_root
 
 pytestmark = [pytest.mark.unit, common.snmp_integration_only]
 
@@ -447,7 +447,7 @@ def test_profile_extends():
     }
 
     with temp_dir() as tmp:
-        with mock_profiles_confd_root(tmp):
+        with mock_profiles_confd_default_root(tmp):
             with open(os.path.join(tmp, 'base.yaml'), 'wb') as f:
                 f.write(yaml.safe_dump(base))
 
@@ -475,7 +475,7 @@ def test_default_profiles():
     }
 
     with temp_dir() as tmp:
-        with mock_profiles_confd_root(tmp):
+        with mock_profiles_confd_default_root(tmp):
             profile_file = os.path.join(tmp, 'profile.yaml')
             with open(profile_file, 'wb') as f:
                 f.write(yaml.safe_dump(profile))
@@ -490,7 +490,7 @@ def test_profile_override():
     }
 
     with temp_dir() as tmp:
-        with mock_profiles_confd_root(tmp):
+        with mock_profiles_confd_default_root(tmp):
             profile_file = os.path.join(tmp, 'generic-device.yaml')
             with open(profile_file, 'wb') as f:
                 f.write(yaml.safe_dump(profile))
@@ -499,6 +499,92 @@ def test_profile_override():
             assert profiles['generic-device'] == {'definition': profile}
 
 
+def test_user_profile_override():
+    default_profile = {
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'metric_default_profile', 'forced_type': 'monotonic_count'}],
+    }
+
+    user_profile = {
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'metric_user_profile', 'forced_type': 'monotonic_count'}],
+    }
+
+    with temp_dir() as tmp:
+        with mock_profiles_confd_default_root(os.path.join(tmp, 'default_profiles')), mock_profiles_confd_user_root(
+            os.path.join(tmp, 'profiles')
+        ):
+            mkdir_p(os.path.join(tmp, 'default_profiles'))
+            mkdir_p(os.path.join(tmp, 'profiles'))
+
+            with open(os.path.join(tmp, 'default_profiles', 'generic-device.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(default_profile))
+            with open(os.path.join(tmp, 'profiles', 'generic-device.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(user_profile))
+
+            profiles = _load_default_profiles()
+            assert profiles['generic-device'] == {'definition': user_profile}
+
+
+def test_profile_extends_with_user_profiles():
+    # type: () -> None
+    default_base = {
+        'metrics': [
+            {'MIB': 'TCP-MIB', 'symbol': 'tcpActiveOpens', 'forced_type': 'monotonic_count'},
+            {'MIB': 'UDP-MIB', 'symbol': 'udpHCInDatagrams', 'forced_type': 'monotonic_count'},
+        ],
+        'metric_tags': [{'MIB': 'SNMPv2-MIB', 'symbol': 'sysName', 'tag': 'snmp_host'}],
+    }
+
+    default_profile1 = {
+        'extends': ['base.yaml'],
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'profile1_metric_default', 'forced_type': 'monotonic_count'}],
+    }
+    default_abstract = {
+        'extends': ['base.yaml'],
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'abstract_metric_default', 'forced_type': 'monotonic_count'}],
+    }
+    user_abstract = {
+        'extends': ['base.yaml'],
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'abstract_metric_user', 'forced_type': 'monotonic_count'}],
+    }
+    user_profile1 = {
+        'extends': ['_abstract.yaml'],
+        'metrics': [{'MIB': 'TCP-MIB', 'symbol': 'profile1_metric_user', 'forced_type': 'monotonic_count'}],
+    }
+
+    with temp_dir() as tmp:
+        with mock_profiles_confd_default_root(os.path.join(tmp, 'default_profiles')), mock_profiles_confd_user_root(
+            os.path.join(tmp, 'profiles')
+        ):
+            mkdir_p(os.path.join(tmp, 'default_profiles'))
+            mkdir_p(os.path.join(tmp, 'profiles'))
+
+            with open(os.path.join(tmp, 'default_profiles', 'base.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(default_base))
+            with open(os.path.join(tmp, 'default_profiles', 'profile1.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(default_profile1))
+            with open(os.path.join(tmp, 'default_profiles', '_abstract.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(default_abstract))
+            with open(os.path.join(tmp, 'profiles', 'profile1.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(user_profile1))
+            with open(os.path.join(tmp, 'profiles', '_abstract.yaml'), 'wb') as f:
+                f.write(yaml.safe_dump(user_abstract))
+
+            definition = {'extends': ['profile1.yaml']}
+
+            recursively_expand_base_profiles(definition)
+
+            assert definition == {
+                'extends': ['profile1.yaml'],
+                'metrics': [
+                    {'MIB': 'TCP-MIB', 'symbol': 'tcpActiveOpens', 'forced_type': 'monotonic_count'},
+                    {'MIB': 'UDP-MIB', 'symbol': 'udpHCInDatagrams', 'forced_type': 'monotonic_count'},
+                    {'MIB': 'TCP-MIB', 'symbol': 'abstract_metric_user', 'forced_type': 'monotonic_count'},
+                    {'MIB': 'TCP-MIB', 'symbol': 'profile1_metric_user', 'forced_type': 'monotonic_count'},
+                ],
+                'metric_tags': [{'MIB': 'SNMPv2-MIB', 'symbol': 'sysName', 'tag': 'snmp_host'}],
+            }
+
+
 def test_discovery_tags():
     """When specifying a tag on discovery, it doesn't make tags leaks between instances."""
     instance = common.generate_instance_config(common.SUPPORTED_METRIC_TYPES)

snmp/tests/utils.py

@@ -3,6 +3,8 @@
 # Licensed under Simplified BSD License (see LICENSE)
 
 import contextlib
+import errno
+import os
 from typing import Iterator  # noqa: F401
 
 import mock
@@ -11,7 +13,25 @@
 
 
 @contextlib.contextmanager
-def mock_profiles_confd_root(root):
+def mock_profiles_confd_default_root(root):
     # type: (str) -> Iterator[None]
-    with mock.patch.object(utils, '_get_profiles_confd_root', return_value=root):
+    with mock.patch.object(utils, '_get_profiles_confd_default_root', return_value=root):
         yield
+
+
+@contextlib.contextmanager
+def mock_profiles_confd_user_root(root):
+    # type: (str) -> Iterator[None]
+    with mock.patch.object(utils, '_get_profiles_confd_user_root', return_value=root):
+        yield
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:  # Python >= 2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        # possibly handle other errno cases here, otherwise finally:
+        else:
+            raise