From 796bdaa46b492bca49d0eafe9706361b39ffef0a Mon Sep 17 00:00:00 2001 From: Jamie Murphy Date: Mon, 12 Sep 2022 21:36:54 +0100 Subject: [PATCH 1/8] netbox: create "connected_devices" function --- salt/pillar/netbox.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index f07c593dd743..08b8aa98e9b2 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -82,6 +82,11 @@ An integer specifying how many results should be returned for each query to the NetBox API. Leaving this unset will use NetBox's default value. +connected_devices: ``False`` + .. versionadded:: 3006.0 + Whether connected_devices key should be populated with device objects. + If set to True it will force `interfaces` to also be true as a dependency + Note that each option you enable can have a detrimental impact on pillar performance, so use them with caution. @@ -777,6 +782,37 @@ def _get_site_details(api_url, minion_id, site_name, site_id, headers): return site_details_ret["dict"] +def _get_connected_devices(api_url, minion_id, interfaces, headers): + log.debug('Retrieving connected devices for "%s"', minion_id) + connected_devices_result = {} + connected_devices_ids = [] + for int_short in interfaces: + if "connected_endpoints" in int_short.keys(): + if int_short["connected_endpoints"]: + for device_short in int_short["connected_endpoints"]: + if not device_short["device"]["id"] in connected_devices_ids: + connected_devices_ids.append(device_short["device"]["id"]) + log.debug("connected_devices_ids: %s", connected_devices_ids) + + for dev_id in connected_devices_ids: + device_url = "{api_url}/{app}/{endpoint}/{dev_id}".format( + api_url=api_url, app="dcim", endpoint="devices", dev_id=dev_id + ) + device_results = [] + device_ret = salt.utils.http.query(device_url, header_dict=headers, decode=True) + if "error" in device_ret: + log.error( + 'API query failed for "%s", status code: %d, error %s', + minion_id, + device_ret["status"], + device_ret["error"], + ) + else: + connected_devices_result[dev_id] = dict(device_ret["dict"]) + + return connected_devices_result + + def _get_site_prefixes( api_url, minion_id, site_name, site_id, headers, api_query_result_limit ): @@ -876,6 +912,13 @@ def ext_pillar(minion_id, pillar, *args, **kwargs): site_prefixes = kwargs.get("site_prefixes", True) proxy_username = kwargs.get("proxy_username", None) proxy_return = kwargs.get("proxy_return", True) + connected_devices = kwargs.get("connected_devices", False) + if connected_devices and not interfaces: + # connected_devices logic requires interfaces to be populated + interfaces = True + log.debug( + "netbox pillar interfaces set to 'True' as connected_devices is 'True'" + ) api_query_result_limit = kwargs.get("api_query_result_limit") ret = {} @@ -951,6 +994,10 @@ def ext_pillar(minion_id, pillar, *args, **kwargs): ret["netbox"]["site"]["prefixes"] = _get_site_prefixes( api_url, minion_id, site_name, site_id, headers, api_query_result_limit ) + if connected_devices: + ret["netbox"]["connected_devices"] = _get_connected_devices( + api_url, minion_id, ret["netbox"]["interfaces"], headers + ) if proxy_return: if ret["netbox"]["platform"]: platform_id = ret["netbox"]["platform"]["id"] From 5a934e1f61726b1ddcfc3e8bb0633aec54e29094 Mon Sep 17 00:00:00 2001 From: Jamie Murphy Date: Mon, 12 Sep 2022 21:47:27 +0100 Subject: [PATCH 2/8] added example output --- salt/pillar/netbox.py | 154 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index 08b8aa98e9b2..e7bc03b0247a 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -525,6 +525,160 @@ tags: custom_fields: config_context: + connected_devices: + ---------- + 1533: + ---------- + airflow: + None + asset_tag: + 001 + cluster: + None + comments: + config_context: + created: + 2022-03-10T00:00:00Z + custom_fields: + device_role: + ---------- + display: + Network switch + id: + 5 + name: + Network switch + slug: + network_switch + url: + https://netbox.example.com/api/dcim/device-roles/5/ + device_type: + ---------- + display: + Nexus 3048 + id: + 40 + manufacturer: + ---------- + display: + Cisco + id: + 1 + name: + Cisco + slug: + cisco + url: + https://netbox.example.com/api/dcim/manufacturers/1/ + model: + Nexus 3048 + slug: + n3k-c3048tp-1ge + url: + https://netbox.example.com/api/dcim/device-types/40/ + display: + another device (001) + face: + ---------- + label: + Front + value: + front + id: + 1533 + last_updated: + 2022-08-22T13:50:15.923868Z + local_context_data: + None + location: + ---------- + _depth: + 2 + display: + Location Name + id: + 2 + name: + Location Name + slug: + location-name + url: + https://netbox.example.com/api/dcim/locations/2 + name: + another device + parent_device: + None + platform: + None + position: + 18.0 + primary_ip: + ---------- + address: + 192.168.1.1/24 + display: + 192.168.1.1/24 + family: + 4 + id: + 1234 + url: + https://netbox.example.com/api/ipam/ip-addresses/1234/ + primary_ip4: + ---------- + address: + 192.168.1.1/24 + display: + 192.168.1.1/24 + family: + 4 + id: + 1234 + url: + https://netbox.example.com/api/ipam/ip-addresses/1234/ + primary_ip6: + None + rack: + ---------- + display: + RackName + id: + 139 + name: + RackName + url: + https://netbox.example.com/api/dcim/racks/139/ + serial: + ABCD12345 + site: + ---------- + display: + SiteName + id: + 2 + name: + SiteName + slug: + sitename + url: + https://netbox.example.com/api/dcim/sites/2/ + status: + ---------- + label: + Active + value: + active + tags: + tenant: + None + url: + https://netbox.example.com/api/dcim/devices/1533/ + vc_position: + None + vc_priority: + None + virtual_chassis: + None created: 2021-02-19 last_updated: From 2666128f38b7df62a9e6b0a69cf3ea6f5464ef89 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" <1613241+ITJamie@users.noreply.github.com> Date: Tue, 13 Sep 2022 11:04:03 +0100 Subject: [PATCH 3/8] set version added to TBD --- salt/pillar/netbox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index e7bc03b0247a..8ab7c5336a03 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -83,7 +83,8 @@ to the NetBox API. Leaving this unset will use NetBox's default value. connected_devices: ``False`` - .. versionadded:: 3006.0 + .. versionadded:: TBD + Whether connected_devices key should be populated with device objects. If set to True it will force `interfaces` to also be true as a dependency From 0a0bc146de0b1f4195a8a00a3ab068f647a02241 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" <1613241+ITJamie@users.noreply.github.com> Date: Tue, 13 Sep 2022 11:12:38 +0100 Subject: [PATCH 4/8] Update netbox.py --- salt/pillar/netbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index 8ab7c5336a03..f8bdadfd1daf 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -83,7 +83,7 @@ to the NetBox API. Leaving this unset will use NetBox's default value. connected_devices: ``False`` - .. versionadded:: TBD + .. versionadded:: 3006 Whether connected_devices key should be populated with device objects. If set to True it will force `interfaces` to also be true as a dependency From 999d9057edc8d13eb21927355bde55eed7e78af0 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" Date: Tue, 27 Sep 2022 16:31:30 +0100 Subject: [PATCH 5/8] add connected_devices tests --- tests/pytests/unit/pillar/test_netbox.py | 244 ++++++++++++++++++++--- 1 file changed, 216 insertions(+), 28 deletions(-) diff --git a/tests/pytests/unit/pillar/test_netbox.py b/tests/pytests/unit/pillar/test_netbox.py index 48463b26b2eb..6d4b20d88dad 100644 --- a/tests/pytests/unit/pillar/test_netbox.py +++ b/tests/pytests/unit/pillar/test_netbox.py @@ -188,8 +188,8 @@ def multiple_device_results(): { "id": 512, "url": "https://netbox.example.com/api/dcim/devices/512/", - "name": "minion1", - "display_name": "minion1", + "name": "minion2", + "display_name": "minion2", "device_type": { "id": 4, "url": "https://netbox.example.com/api/dcim/device-types/4/", @@ -490,9 +490,24 @@ def device_interface_results(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": [ + { + "id": 170, + "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", + "display": "GigabitEthernet1/0/1", + "device": { + "id": 512, + "url": "https://demo.netbox.dev/api/dcim/devices/512/", + "display": "minion2", + "name": "minion2" + }, + "name": "GigabitEthernet1/0/1", + "cable": 35, + "_occupied": True + } + ], + "connected_endpoints_type": "dcim.interface", + "connected_endpoints_reachable": True, "tags": [], "count_ipaddresses": 1, }, @@ -520,9 +535,9 @@ def device_interface_results(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": None, + "connected_endpoints_type": None, + "connected_endpoints_reachable": None, "tags": [], "count_ipaddresses": 1, }, @@ -552,9 +567,18 @@ def device_interfaces_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + 'connected_endpoints': [{'_occupied': True, + 'cable': 35, + 'device': {'display': 'minion2', + 'id': 512, + 'name': 'minion2', + 'url': 'https://demo.netbox.dev/api/dcim/devices/512/'}, + 'display': 'GigabitEthernet1/0/1', + 'id': 170, + 'name': 'GigabitEthernet1/0/1', + 'url': 'https://demo.netbox.dev/api/dcim/interfaces/512/'}], + 'connected_endpoints_reachable': True, + 'connected_endpoints_type': 'dcim.interface', "tags": [], "count_ipaddresses": 1, }, @@ -576,9 +600,9 @@ def device_interfaces_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": None, + "connected_endpoints_type": None, + "connected_endpoints_reachable": None, "tags": [], "count_ipaddresses": 1, }, @@ -857,9 +881,24 @@ def device_interfaces_ip_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": [ + { + "id": 170, + "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", + "display": "GigabitEthernet1/0/1", + "device": { + "id": 512, + "url": "https://demo.netbox.dev/api/dcim/devices/512/", + "display": "minion2", + "name": "minion2" + }, + "name": "GigabitEthernet1/0/1", + "cable": 35, + "_occupied": True + } + ], + "connected_endpoints_type": "dcim.interface", + "connected_endpoints_reachable": True, "tags": [], "count_ipaddresses": 1, }, @@ -901,9 +940,9 @@ def device_interfaces_ip_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": None, + "connected_endpoints_type": None, + "connected_endpoints_reachable": None, "tags": [], "count_ipaddresses": 1, }, @@ -1217,9 +1256,24 @@ def pillar_results(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": [ + { + "id": 170, + "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", + "display": "GigabitEthernet1/0/1", + "device": { + "id": 512, + "url": "https://demo.netbox.dev/api/dcim/devices/512/", + "display": "minion2", + "name": "minion2" + }, + "name": "GigabitEthernet1/0/1", + "cable": 35, + "_occupied": True + } + ], + "connected_endpoints_type": "dcim.interface", + "connected_endpoints_reachable": True, "tags": [], "count_ipaddresses": 1, }, @@ -1261,9 +1315,9 @@ def pillar_results(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoint": None, - "connected_endpoint_type": None, - "connected_endpoint_reachable": None, + "connected_endpoints": None, + "connected_endpoints_type": None, + "connected_endpoints_reachable": None, "tags": [], "count_ipaddresses": 1, }, @@ -1371,12 +1425,141 @@ def pillar_results(): "tags": [], "custom_fields": {}, "config_context": {}, + 'connected_devices': {'asset_tag': None, + 'cluster': None, + 'comments': '', + 'config_context': {}, + 'created': '2021-02-19', + 'custom_fields': {}, + 'device_role': {'id': 45, + 'name': 'Network', + 'slug': 'network', + 'url': 'https://netbox.example.com/api/dcim/device-roles/45/'}, + 'device_type': {'display_name': 'Cisco ' + 'ISR2901', + 'id': 4, + 'manufacturer': {'id': 1, + 'name': 'Cisco', + 'slug': 'cisco', + 'url': 'https://netbox.example.com/api/dcim/manufacturers/1/'}, + 'model': 'ISR2901', + 'slug': 'isr2901', + 'url': 'https://netbox.example.com/api/dcim/device-types/4/'}, + 'display_name': 'minion2', + 'face': None, + 'id': 512, + 'last_updated': '2021-02-19T06:12:04.171105Z', + 'local_context_data': None, + 'name': 'minion2', + 'node_type': 'device', + 'parent_device': None, + 'platform': {'id': 1, + 'name': 'Cisco IOS', + 'slug': 'ios', + 'url': 'https://netbox.example.com/api/dcim/platforms/1/'}, + 'position': None, + 'primary_ip': {'address': '192.0.2.3/24', + 'family': 4, + 'id': 1150, + 'url': 'https://netbox.example.com/api/ipam/ip-addresses/1150/'}, + 'primary_ip4': {'address': '192.0.2.3/24', + 'family': 4, + 'id': 1150, + 'url': 'https://netbox.example.com/api/ipam/ip-addresses/1150/'}, + 'primary_ip6': None, + 'rack': None, + 'serial': '', + 'site': {'id': 18, + 'name': 'Site 1', + 'slug': 'site1', + 'url': 'https://netbox.example.com/api/dcim/sites/18/'}, + 'status': {'label': 'Active', + 'value': 'active'}, + 'tags': [], + 'tenant': None, + 'url': 'https://netbox.example.com/api/dcim/devices/512/', + 'vc_position': None, + 'vc_priority': None, + 'virtual_chassis': None}, "created": "2021-02-19", "last_updated": "2021-02-19T06:12:04.171105Z", }, "proxy": {"host": "192.0.2.1", "driver": "ios", "proxytype": "napalm"}, } +@pytest.fixture +def connected_devices_results(): + return { + "id": 512, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "name": "minion2", + "display_name": "minion2", + "device_type": { + "id": 4, + "url": "https://netbox.example.com/api/dcim/device-types/4/", + "manufacturer": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + "name": "Cisco", + "slug": "cisco", + }, + "model": "ISR2901", + "slug": "isr2901", + "display_name": "Cisco ISR2901", + }, + "device_role": { + "id": 45, + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + "name": "Network", + "slug": "network", + }, + "node_type": "device", + "tenant": None, + "platform": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/platforms/1/", + "name": "Cisco IOS", + "slug": "ios", + }, + "serial": "", + "asset_tag": None, + "site": { + "id": 18, + "url": "https://netbox.example.com/api/dcim/sites/18/", + "name": "Site 1", + "slug": "site1", + }, + "rack": None, + "position": None, + "face": None, + "parent_device": None, + "status": {"value": "active", "label": "Active"}, + "primary_ip": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip4": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip6": None, + "cluster": None, + "virtual_chassis": None, + "vc_position": None, + "vc_priority": None, + "comments": "", + "local_context_data": None, + "tags": [], + "custom_fields": {}, + "config_context": {}, + "created": "2021-02-19", + "last_updated": "2021-02-19T06:12:04.171105Z", + } + def test_when_minion_id_is_star_then_result_should_be_empty_dict(default_kwargs): expected_result = {} @@ -1968,6 +2151,7 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( site_prefixes, proxy_details, pillar_results, + connected_devices_results ): expected_result = pillar_results @@ -1978,6 +2162,7 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( default_kwargs["site_details"] = True default_kwargs["site_prefixes"] = True default_kwargs["proxy_return"] = True + default_kwargs["connected_devices"] = True with patch("salt.pillar.netbox._get_devices", autospec=True) as get_devices, patch( "salt.pillar.netbox._get_virtual_machines", autospec=True @@ -1991,8 +2176,10 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( "salt.pillar.netbox._get_site_prefixes", autospec=True ) as get_site_prefixes, patch( "salt.pillar.netbox._get_proxy_details", autospec=True - ) as get_proxy_details: - + ) as get_proxy_details, patch( + "salt.pillar.netbox._get_connected_devices", autospec=True + ) as get_connected_decvices: + get_devices.return_value = device_results["dict"]["results"] get_virtual_machines.return_value = no_results["dict"]["results"] get_interfaces.return_value = device_interfaces_list @@ -2000,6 +2187,7 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( get_site_details.return_value = site_results["dict"] get_site_prefixes.return_value = site_prefixes get_proxy_details.return_value = proxy_details + get_connected_decvices.return_value = connected_devices_results actual_result = netbox.ext_pillar(**default_kwargs) From 46feb062107eadf6cb19a9b3619b8f6aeaef9177 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" Date: Tue, 27 Sep 2022 16:31:48 +0100 Subject: [PATCH 6/8] blacken --- tests/pytests/unit/pillar/test_netbox.py | 341 ++++++++++++----------- 1 file changed, 181 insertions(+), 160 deletions(-) diff --git a/tests/pytests/unit/pillar/test_netbox.py b/tests/pytests/unit/pillar/test_netbox.py index 6d4b20d88dad..e9cc281effe2 100644 --- a/tests/pytests/unit/pillar/test_netbox.py +++ b/tests/pytests/unit/pillar/test_netbox.py @@ -499,11 +499,11 @@ def device_interface_results(): "id": 512, "url": "https://demo.netbox.dev/api/dcim/devices/512/", "display": "minion2", - "name": "minion2" + "name": "minion2", }, "name": "GigabitEthernet1/0/1", "cable": 35, - "_occupied": True + "_occupied": True, } ], "connected_endpoints_type": "dcim.interface", @@ -567,18 +567,24 @@ def device_interfaces_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - 'connected_endpoints': [{'_occupied': True, - 'cable': 35, - 'device': {'display': 'minion2', - 'id': 512, - 'name': 'minion2', - 'url': 'https://demo.netbox.dev/api/dcim/devices/512/'}, - 'display': 'GigabitEthernet1/0/1', - 'id': 170, - 'name': 'GigabitEthernet1/0/1', - 'url': 'https://demo.netbox.dev/api/dcim/interfaces/512/'}], - 'connected_endpoints_reachable': True, - 'connected_endpoints_type': 'dcim.interface', + "connected_endpoints": [ + { + "_occupied": True, + "cable": 35, + "device": { + "display": "minion2", + "id": 512, + "name": "minion2", + "url": "https://demo.netbox.dev/api/dcim/devices/512/", + }, + "display": "GigabitEthernet1/0/1", + "id": 170, + "name": "GigabitEthernet1/0/1", + "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", + } + ], + "connected_endpoints_reachable": True, + "connected_endpoints_type": "dcim.interface", "tags": [], "count_ipaddresses": 1, }, @@ -881,24 +887,24 @@ def device_interfaces_ip_list(): "cable": None, "cable_peer": None, "cable_peer_type": None, - "connected_endpoints": [ - { - "id": 170, - "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", - "display": "GigabitEthernet1/0/1", - "device": { - "id": 512, - "url": "https://demo.netbox.dev/api/dcim/devices/512/", - "display": "minion2", - "name": "minion2" - }, - "name": "GigabitEthernet1/0/1", - "cable": 35, - "_occupied": True - } - ], - "connected_endpoints_type": "dcim.interface", - "connected_endpoints_reachable": True, + "connected_endpoints": [ + { + "id": 170, + "url": "https://demo.netbox.dev/api/dcim/interfaces/512/", + "display": "GigabitEthernet1/0/1", + "device": { + "id": 512, + "url": "https://demo.netbox.dev/api/dcim/devices/512/", + "display": "minion2", + "name": "minion2", + }, + "name": "GigabitEthernet1/0/1", + "cable": 35, + "_occupied": True, + } + ], + "connected_endpoints_type": "dcim.interface", + "connected_endpoints_reachable": True, "tags": [], "count_ipaddresses": 1, }, @@ -1265,11 +1271,11 @@ def pillar_results(): "id": 512, "url": "https://demo.netbox.dev/api/dcim/devices/512/", "display": "minion2", - "name": "minion2" + "name": "minion2", }, "name": "GigabitEthernet1/0/1", "cable": 35, - "_occupied": True + "_occupied": True, } ], "connected_endpoints_type": "dcim.interface", @@ -1425,139 +1431,154 @@ def pillar_results(): "tags": [], "custom_fields": {}, "config_context": {}, - 'connected_devices': {'asset_tag': None, - 'cluster': None, - 'comments': '', - 'config_context': {}, - 'created': '2021-02-19', - 'custom_fields': {}, - 'device_role': {'id': 45, - 'name': 'Network', - 'slug': 'network', - 'url': 'https://netbox.example.com/api/dcim/device-roles/45/'}, - 'device_type': {'display_name': 'Cisco ' - 'ISR2901', - 'id': 4, - 'manufacturer': {'id': 1, - 'name': 'Cisco', - 'slug': 'cisco', - 'url': 'https://netbox.example.com/api/dcim/manufacturers/1/'}, - 'model': 'ISR2901', - 'slug': 'isr2901', - 'url': 'https://netbox.example.com/api/dcim/device-types/4/'}, - 'display_name': 'minion2', - 'face': None, - 'id': 512, - 'last_updated': '2021-02-19T06:12:04.171105Z', - 'local_context_data': None, - 'name': 'minion2', - 'node_type': 'device', - 'parent_device': None, - 'platform': {'id': 1, - 'name': 'Cisco IOS', - 'slug': 'ios', - 'url': 'https://netbox.example.com/api/dcim/platforms/1/'}, - 'position': None, - 'primary_ip': {'address': '192.0.2.3/24', - 'family': 4, - 'id': 1150, - 'url': 'https://netbox.example.com/api/ipam/ip-addresses/1150/'}, - 'primary_ip4': {'address': '192.0.2.3/24', - 'family': 4, - 'id': 1150, - 'url': 'https://netbox.example.com/api/ipam/ip-addresses/1150/'}, - 'primary_ip6': None, - 'rack': None, - 'serial': '', - 'site': {'id': 18, - 'name': 'Site 1', - 'slug': 'site1', - 'url': 'https://netbox.example.com/api/dcim/sites/18/'}, - 'status': {'label': 'Active', - 'value': 'active'}, - 'tags': [], - 'tenant': None, - 'url': 'https://netbox.example.com/api/dcim/devices/512/', - 'vc_position': None, - 'vc_priority': None, - 'virtual_chassis': None}, + "connected_devices": { + "asset_tag": None, + "cluster": None, + "comments": "", + "config_context": {}, + "created": "2021-02-19", + "custom_fields": {}, + "device_role": { + "id": 45, + "name": "Network", + "slug": "network", + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + }, + "device_type": { + "display_name": "Cisco " "ISR2901", + "id": 4, + "manufacturer": { + "id": 1, + "name": "Cisco", + "slug": "cisco", + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + }, + "model": "ISR2901", + "slug": "isr2901", + "url": "https://netbox.example.com/api/dcim/device-types/4/", + }, + "display_name": "minion2", + "face": None, + "id": 512, + "last_updated": "2021-02-19T06:12:04.171105Z", + "local_context_data": None, + "name": "minion2", + "node_type": "device", + "parent_device": None, + "platform": { + "id": 1, + "name": "Cisco IOS", + "slug": "ios", + "url": "https://netbox.example.com/api/dcim/platforms/1/", + }, + "position": None, + "primary_ip": { + "address": "192.0.2.3/24", + "family": 4, + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + }, + "primary_ip4": { + "address": "192.0.2.3/24", + "family": 4, + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + }, + "primary_ip6": None, + "rack": None, + "serial": "", + "site": { + "id": 18, + "name": "Site 1", + "slug": "site1", + "url": "https://netbox.example.com/api/dcim/sites/18/", + }, + "status": {"label": "Active", "value": "active"}, + "tags": [], + "tenant": None, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "vc_position": None, + "vc_priority": None, + "virtual_chassis": None, + }, "created": "2021-02-19", "last_updated": "2021-02-19T06:12:04.171105Z", }, "proxy": {"host": "192.0.2.1", "driver": "ios", "proxytype": "napalm"}, } + @pytest.fixture def connected_devices_results(): return { - "id": 512, - "url": "https://netbox.example.com/api/dcim/devices/512/", - "name": "minion2", - "display_name": "minion2", - "device_type": { - "id": 4, - "url": "https://netbox.example.com/api/dcim/device-types/4/", - "manufacturer": { - "id": 1, - "url": "https://netbox.example.com/api/dcim/manufacturers/1/", - "name": "Cisco", - "slug": "cisco", - }, - "model": "ISR2901", - "slug": "isr2901", - "display_name": "Cisco ISR2901", - }, - "device_role": { - "id": 45, - "url": "https://netbox.example.com/api/dcim/device-roles/45/", - "name": "Network", - "slug": "network", - }, - "node_type": "device", - "tenant": None, - "platform": { - "id": 1, - "url": "https://netbox.example.com/api/dcim/platforms/1/", - "name": "Cisco IOS", - "slug": "ios", - }, - "serial": "", - "asset_tag": None, - "site": { - "id": 18, - "url": "https://netbox.example.com/api/dcim/sites/18/", - "name": "Site 1", - "slug": "site1", - }, - "rack": None, - "position": None, - "face": None, - "parent_device": None, - "status": {"value": "active", "label": "Active"}, - "primary_ip": { - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - "family": 4, - "address": "192.0.2.3/24", - }, - "primary_ip4": { - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - "family": 4, - "address": "192.0.2.3/24", - }, - "primary_ip6": None, - "cluster": None, - "virtual_chassis": None, - "vc_position": None, - "vc_priority": None, - "comments": "", - "local_context_data": None, - "tags": [], - "custom_fields": {}, - "config_context": {}, - "created": "2021-02-19", - "last_updated": "2021-02-19T06:12:04.171105Z", + "id": 512, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "name": "minion2", + "display_name": "minion2", + "device_type": { + "id": 4, + "url": "https://netbox.example.com/api/dcim/device-types/4/", + "manufacturer": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + "name": "Cisco", + "slug": "cisco", + }, + "model": "ISR2901", + "slug": "isr2901", + "display_name": "Cisco ISR2901", + }, + "device_role": { + "id": 45, + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + "name": "Network", + "slug": "network", + }, + "node_type": "device", + "tenant": None, + "platform": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/platforms/1/", + "name": "Cisco IOS", + "slug": "ios", + }, + "serial": "", + "asset_tag": None, + "site": { + "id": 18, + "url": "https://netbox.example.com/api/dcim/sites/18/", + "name": "Site 1", + "slug": "site1", + }, + "rack": None, + "position": None, + "face": None, + "parent_device": None, + "status": {"value": "active", "label": "Active"}, + "primary_ip": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip4": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip6": None, + "cluster": None, + "virtual_chassis": None, + "vc_position": None, + "vc_priority": None, + "comments": "", + "local_context_data": None, + "tags": [], + "custom_fields": {}, + "config_context": {}, + "created": "2021-02-19", + "last_updated": "2021-02-19T06:12:04.171105Z", } @@ -2151,7 +2172,7 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( site_prefixes, proxy_details, pillar_results, - connected_devices_results + connected_devices_results, ): expected_result = pillar_results @@ -2179,7 +2200,7 @@ def test_when_we_retrieve_everything_successfully_then_return_dict( ) as get_proxy_details, patch( "salt.pillar.netbox._get_connected_devices", autospec=True ) as get_connected_decvices: - + get_devices.return_value = device_results["dict"]["results"] get_virtual_machines.return_value = no_results["dict"]["results"] get_interfaces.return_value = device_interfaces_list From fc386902effcfc01ac8008129f0ccc23004fc8c0 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" Date: Tue, 27 Sep 2022 16:37:41 +0100 Subject: [PATCH 7/8] Create 62761.added --- changelog/62761.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/62761.added diff --git a/changelog/62761.added b/changelog/62761.added new file mode 100644 index 000000000000..29944baaf8bb --- /dev/null +++ b/changelog/62761.added @@ -0,0 +1 @@ +Added "connected_devices" feature to netbox pillar module. It contains extra information about devices connected to the minion \ No newline at end of file From 75869b070c3da75677c3a37cfb4cb885dc8c4337 Mon Sep 17 00:00:00 2001 From: "Jamie (Bear) Murphy" Date: Tue, 27 Sep 2022 17:11:39 +0100 Subject: [PATCH 8/8] add test to netbox pillar connected_endpoints --- salt/pillar/netbox.py | 9 +- tests/pytests/unit/pillar/test_netbox.py | 367 +++++++++++++++-------- 2 files changed, 241 insertions(+), 135 deletions(-) diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index f8bdadfd1daf..e2c0ab7d70a0 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -528,7 +528,7 @@ config_context: connected_devices: ---------- - 1533: + 512: ---------- airflow: None @@ -546,7 +546,7 @@ display: Network switch id: - 5 + 512 name: Network switch slug: @@ -945,7 +945,10 @@ def _get_connected_devices(api_url, minion_id, interfaces, headers): if "connected_endpoints" in int_short.keys(): if int_short["connected_endpoints"]: for device_short in int_short["connected_endpoints"]: - if not device_short["device"]["id"] in connected_devices_ids: + if ( + "device" in device_short.keys() + and not device_short["device"]["id"] in connected_devices_ids + ): connected_devices_ids.append(device_short["device"]["id"]) log.debug("connected_devices_ids: %s", connected_devices_ids) diff --git a/tests/pytests/unit/pillar/test_netbox.py b/tests/pytests/unit/pillar/test_netbox.py index e9cc281effe2..727cea628ed9 100644 --- a/tests/pytests/unit/pillar/test_netbox.py +++ b/tests/pytests/unit/pillar/test_netbox.py @@ -260,6 +260,82 @@ def multiple_device_results(): } +@pytest.fixture +def secondary_device_result(): + return { + "dict": { + "id": 512, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "name": "minion2", + "display_name": "minion2", + "device_type": { + "id": 4, + "url": "https://netbox.example.com/api/dcim/device-types/4/", + "manufacturer": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + "name": "Cisco", + "slug": "cisco", + }, + "model": "ISR2901", + "slug": "isr2901", + "display_name": "Cisco ISR2901", + }, + "device_role": { + "id": 45, + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + "name": "Network", + "slug": "network", + }, + "node_type": "device", + "tenant": None, + "platform": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/platforms/1/", + "name": "Cisco IOS", + "slug": "ios", + }, + "serial": "", + "asset_tag": None, + "site": { + "id": 18, + "url": "https://netbox.example.com/api/dcim/sites/18/", + "name": "Site 1", + "slug": "site1", + }, + "rack": None, + "position": None, + "face": None, + "parent_device": None, + "status": {"value": "active", "label": "Active"}, + "primary_ip": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip4": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip6": None, + "cluster": None, + "virtual_chassis": None, + "vc_position": None, + "vc_priority": None, + "comments": "", + "local_context_data": None, + "tags": [], + "custom_fields": {}, + "config_context": {}, + "created": "2021-02-19", + "last_updated": "2021-02-19T06:12:04.171105Z", + } + } + + @pytest.fixture def virtual_machine_results(): return { @@ -1432,74 +1508,76 @@ def pillar_results(): "custom_fields": {}, "config_context": {}, "connected_devices": { - "asset_tag": None, - "cluster": None, - "comments": "", - "config_context": {}, - "created": "2021-02-19", - "custom_fields": {}, - "device_role": { - "id": 45, - "name": "Network", - "slug": "network", - "url": "https://netbox.example.com/api/dcim/device-roles/45/", - }, - "device_type": { - "display_name": "Cisco " "ISR2901", - "id": 4, - "manufacturer": { + 512: { + "asset_tag": None, + "cluster": None, + "comments": "", + "config_context": {}, + "created": "2021-02-19", + "custom_fields": {}, + "device_role": { + "id": 45, + "name": "Network", + "slug": "network", + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + }, + "device_type": { + "display_name": "Cisco " "ISR2901", + "id": 4, + "manufacturer": { + "id": 1, + "name": "Cisco", + "slug": "cisco", + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + }, + "model": "ISR2901", + "slug": "isr2901", + "url": "https://netbox.example.com/api/dcim/device-types/4/", + }, + "display_name": "minion2", + "face": None, + "id": 512, + "last_updated": "2021-02-19T06:12:04.171105Z", + "local_context_data": None, + "name": "minion2", + "node_type": "device", + "parent_device": None, + "platform": { "id": 1, - "name": "Cisco", - "slug": "cisco", - "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + "name": "Cisco IOS", + "slug": "ios", + "url": "https://netbox.example.com/api/dcim/platforms/1/", }, - "model": "ISR2901", - "slug": "isr2901", - "url": "https://netbox.example.com/api/dcim/device-types/4/", - }, - "display_name": "minion2", - "face": None, - "id": 512, - "last_updated": "2021-02-19T06:12:04.171105Z", - "local_context_data": None, - "name": "minion2", - "node_type": "device", - "parent_device": None, - "platform": { - "id": 1, - "name": "Cisco IOS", - "slug": "ios", - "url": "https://netbox.example.com/api/dcim/platforms/1/", - }, - "position": None, - "primary_ip": { - "address": "192.0.2.3/24", - "family": 4, - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - }, - "primary_ip4": { - "address": "192.0.2.3/24", - "family": 4, - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - }, - "primary_ip6": None, - "rack": None, - "serial": "", - "site": { - "id": 18, - "name": "Site 1", - "slug": "site1", - "url": "https://netbox.example.com/api/dcim/sites/18/", - }, - "status": {"label": "Active", "value": "active"}, - "tags": [], - "tenant": None, - "url": "https://netbox.example.com/api/dcim/devices/512/", - "vc_position": None, - "vc_priority": None, - "virtual_chassis": None, + "position": None, + "primary_ip": { + "address": "192.0.2.3/24", + "family": 4, + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + }, + "primary_ip4": { + "address": "192.0.2.3/24", + "family": 4, + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + }, + "primary_ip6": None, + "rack": None, + "serial": "", + "site": { + "id": 18, + "name": "Site 1", + "slug": "site1", + "url": "https://netbox.example.com/api/dcim/sites/18/", + }, + "status": {"label": "Active", "value": "active"}, + "tags": [], + "tenant": None, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "vc_position": None, + "vc_priority": None, + "virtual_chassis": None, + } }, "created": "2021-02-19", "last_updated": "2021-02-19T06:12:04.171105Z", @@ -1511,74 +1589,76 @@ def pillar_results(): @pytest.fixture def connected_devices_results(): return { - "id": 512, - "url": "https://netbox.example.com/api/dcim/devices/512/", - "name": "minion2", - "display_name": "minion2", - "device_type": { - "id": 4, - "url": "https://netbox.example.com/api/dcim/device-types/4/", - "manufacturer": { + 512: { + "id": 512, + "url": "https://netbox.example.com/api/dcim/devices/512/", + "name": "minion2", + "display_name": "minion2", + "device_type": { + "id": 4, + "url": "https://netbox.example.com/api/dcim/device-types/4/", + "manufacturer": { + "id": 1, + "url": "https://netbox.example.com/api/dcim/manufacturers/1/", + "name": "Cisco", + "slug": "cisco", + }, + "model": "ISR2901", + "slug": "isr2901", + "display_name": "Cisco ISR2901", + }, + "device_role": { + "id": 45, + "url": "https://netbox.example.com/api/dcim/device-roles/45/", + "name": "Network", + "slug": "network", + }, + "node_type": "device", + "tenant": None, + "platform": { "id": 1, - "url": "https://netbox.example.com/api/dcim/manufacturers/1/", - "name": "Cisco", - "slug": "cisco", + "url": "https://netbox.example.com/api/dcim/platforms/1/", + "name": "Cisco IOS", + "slug": "ios", }, - "model": "ISR2901", - "slug": "isr2901", - "display_name": "Cisco ISR2901", - }, - "device_role": { - "id": 45, - "url": "https://netbox.example.com/api/dcim/device-roles/45/", - "name": "Network", - "slug": "network", - }, - "node_type": "device", - "tenant": None, - "platform": { - "id": 1, - "url": "https://netbox.example.com/api/dcim/platforms/1/", - "name": "Cisco IOS", - "slug": "ios", - }, - "serial": "", - "asset_tag": None, - "site": { - "id": 18, - "url": "https://netbox.example.com/api/dcim/sites/18/", - "name": "Site 1", - "slug": "site1", - }, - "rack": None, - "position": None, - "face": None, - "parent_device": None, - "status": {"value": "active", "label": "Active"}, - "primary_ip": { - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - "family": 4, - "address": "192.0.2.3/24", - }, - "primary_ip4": { - "id": 1150, - "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", - "family": 4, - "address": "192.0.2.3/24", - }, - "primary_ip6": None, - "cluster": None, - "virtual_chassis": None, - "vc_position": None, - "vc_priority": None, - "comments": "", - "local_context_data": None, - "tags": [], - "custom_fields": {}, - "config_context": {}, - "created": "2021-02-19", - "last_updated": "2021-02-19T06:12:04.171105Z", + "serial": "", + "asset_tag": None, + "site": { + "id": 18, + "url": "https://netbox.example.com/api/dcim/sites/18/", + "name": "Site 1", + "slug": "site1", + }, + "rack": None, + "position": None, + "face": None, + "parent_device": None, + "status": {"value": "active", "label": "Active"}, + "primary_ip": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip4": { + "id": 1150, + "url": "https://netbox.example.com/api/ipam/ip-addresses/1150/", + "family": 4, + "address": "192.0.2.3/24", + }, + "primary_ip6": None, + "cluster": None, + "virtual_chassis": None, + "vc_position": None, + "vc_priority": None, + "comments": "", + "local_context_data": None, + "tags": [], + "custom_fields": {}, + "config_context": {}, + "created": "2021-02-19", + "last_updated": "2021-02-19T06:12:04.171105Z", + } } @@ -1840,6 +1920,29 @@ def test_when_we_retrieve_device_interface_ips_then_return_list( assert actual_result == expected_result +def test_connected_endpoints( + default_kwargs, + headers, + connected_devices_results, + device_interfaces_list, + secondary_device_result, +): + + expected_result = connected_devices_results + + with patch("salt.utils.http.query", autospec=True) as query: + query.return_value = secondary_device_result + + actual_result = netbox._get_connected_devices( + default_kwargs["api_url"], + default_kwargs["minion_id"], + device_interfaces_list, + headers, + ) + + assert actual_result == expected_result + + def test_when_we_retrieve_device_interface_ips_and_get_http_error_then_return_empty_list( default_kwargs, headers, http_error ):