Update cv_facts to speed up process (#147)

Rewrite def facts_configlets to use get_configlets_and_mappers function
and reduce number of API call.

No change in inputs and outputs.

Test cases:
-----------
- Manual cv_facts tasks
- AVD integration

Execution time enhacement:
--------------------------
- Original implementation: 6 sec
- New version: 0.3 sec

test setup:
------------
8 devices / 16 configlets / 10 containers

Execution logs:
---------------
2020-03-18 18:26:09,700 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:269) - Collecting facts v2
2020-03-18 18:26:10,017 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:276) - Devices part of facts, using this version
2020-03-18 18:26:10,017 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:283) - Containers part of facts, using this version
2020-03-18 18:26:10,017 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet leaf02-baseline
2020-03-18 18:26:10,017 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet RADIUS_LAB
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-LEAF2B
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet Builder_LEAFs
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet spine01-baseline
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-SPINE2
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-LEAF1B
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-SPINE1
2020-03-18 18:26:10,018 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet GLOBAL-ALIASES
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-L2LEAF1A
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet leaf01-baseline
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet veos-01
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-L2LEAF2A
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-LEAF2A
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet SYS_TelemetryBuilderV3
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet DC1-AVD_DC1-LEAF1A
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet data.device.leaf01
2020-03-18 18:26:10,019 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:296) - building list of mapping with devices and containers for configlet template.interfaces
2020-03-18 18:26:10,020 - arista.cvp.cv_facts: INFO - func: facts_configlets (L:317) - All configlets facts collected

ansible_collections/arista/cvp/plugins/module_utils/tools_inventory.py

@@ -0,0 +1,49 @@
+import logging
+import ansible_collections.arista.cvp.plugins.module_utils.logger
+
+MODULE_LOGGER = logging.getLogger('arista.cvp.tools_inventory')
+
+def find_hostname_by_mac(inventory, mac_address):
+    """
+    Function to get device hostname based on System Mac Address.
+
+    Parameters
+    ----------
+    inventory : list
+        Inventory list extracted from CVP.
+    mac_address : string
+        Mac address to search
+
+    Returns
+    -------
+    string
+        Device hostname. Default None if not found
+    """
+    for device in inventory:
+        if 'systemMacAddress' in device:
+            if device['systemMacAddress'] == mac_address:
+                return device['name']
+    return None
+
+
+def find_containerName_by_containerId(containers_list, container_id):
+    """
+    Function to get containername based on container ID.
+
+    Parameters
+    ----------
+    containers_list : list
+        Containers list extracted from CVP.
+    container_id : string
+        ID of the container to search
+
+    Returns
+    -------
+    string
+        Container name. Default None if not found
+    """
+    for container in containers_list:
+        if 'Key' in container:
+            if container['Key'] == container_id:
+                return container['Name']
+    return None

ansible_collections/arista/cvp/plugins/modules/cv_facts.py

@@ -32,6 +32,10 @@
 from ansible.module_utils.connection import Connection, ConnectionError
 from ansible_collections.arista.cvp.plugins.module_utils.cv_client import CvpClient
 from ansible_collections.arista.cvp.plugins.module_utils.cv_client_errors import CvpLoginError, CvpApiError
+from ansible_collections.arista.cvp.plugins.module_utils.tools_inventory import (
+    find_hostname_by_mac,
+    find_containerName_by_containerId
+    )
 import ansible_collections.arista.cvp.plugins.module_utils.logger
 
 DOCUMENTATION = r'''
@@ -197,9 +201,9 @@ def facts_devices(module, facts):
     return facts
 
 
-def facts_configlets(module, facts):
+def facts_configlets_v1(module, facts):
     """
-    Collect facts of all configlets.
+    DEPRECATED - Collect facts of all configlets.
 
     Parameters
     ----------
@@ -217,25 +221,102 @@ def facts_configlets(module, facts):
     """
 
     facts['configlets'] = []
+    MODULE_LOGGER.info('Collecting facts v1')
     configlets = module.client.api.get_configlets()['data']
     # Reduce configlet data to required fields
     for configlet in configlets:
         MODULE_LOGGER.debug('  -> Working on %s', configlet['name'])
         # Get list of devices attached to configlet.
         configlet['devices'] = []
+        MODULE_LOGGER.debug('  -> collecting list of attached devices to configlet: %s', str(configlet))
         applied_devices = module.client.api.get_devices_by_configlet(configlet['name'])
         for device in applied_devices['data']:
             configlet['devices'].append(device['hostName'])
 
         # Get list of containers attached to configlet.
         configlet['containers'] = []
-        applied_containers = module.client.api.get_containers_by_configlet(configlet['name'])
+        MODULE_LOGGER.debug(
+            '  -> collecting list of attached containers to configlet: %s', str(configlet))
+        applied_containers = module.client.api.get_containers_by_configlet(
+            configlet['name'])
         for container in applied_containers['data']:
             configlet['containers'].append(container['containerName'])
 
         # Add configlet to facts list
         facts['configlets'].append(configlet)
+    MODULE_LOGGER.info('All configlet facts collected')
+
+    return facts
+
+
+def facts_configlets(module, facts):
+    """
+    WORK IN PROGRESS - Collect facts of all configlets.
+
+    Parameters
+    ----------
+    module : AnsibleModule
+        Ansible module with parameters and instances
+    facts : dict
+        Fact dictionary where configlets information will be inserted.
+    debug : bool, optional
+        Activate debug logging, by default False
 
+    Returns
+    -------
+    dict
+        facts with configlets content added.
+    """
+    facts['configlets'] = []
+    MODULE_LOGGER.info('Collecting facts v2')
+    configlets_and_mappers = module.client.api.get_configlets_and_mappers()[
+        'data']
+    # Load data to match ID with human readable name
+    inventory = list()
+    containers = list()
+    if 'devices' in facts:
+        MODULE_LOGGER.info('Devices part of facts, using cached version')
+        inventory = facts['devices']
+    else:
+        MODULE_LOGGER.warning(
+            'Devices not part of facts, collecting CV version')
+        inventory = module.client.api.get_inventory()
+    if 'containers' in facts:
+        MODULE_LOGGER.info('Containers part of facts, using cached version')
+        containers = facts['containers']
+    else:
+        MODULE_LOGGER.warning(
+            'Containers not part of facts, collecting CV version')
+        containers = module.client.api.get_containers()['data']
+
+    # Create list of configlets
+    if 'configlets' in configlets_and_mappers:
+        for configlet in configlets_and_mappers['configlets']:
+            configlet['devices'] = list()
+            configlet['containers'] = list()
+            # Parse mapper section to locate potential mappings to devices and containers.
+            MODULE_LOGGER.info('building list of mapping with devices and containers for configlet %s', str(configlet['name']))
+            for mapper in configlets_and_mappers['configletMappers']:
+                # If mapper is tied to our configlet
+                if mapper['configletId'] == configlet['key']:
+                    # If mapper is for device
+                    if mapper['type'] == 'netelement':
+                        device_hostname = find_hostname_by_mac(inventory=inventory, mac_address=mapper['objectId'])
+                        if device_hostname is not None:
+                            MODULE_LOGGER.debug('found mapping to device %s', str(device_hostname))
+                            configlet['devices'].append(device_hostname)
+                    # If mapper is for container
+                    if mapper['type'] == 'container':
+                        container_name = find_containerName_by_containerId(containers_list=containers,
+                                                                            container_id=mapper['objectId'])
+                        if container_name is not None:
+                            MODULE_LOGGER.debug(
+                                'found mapping to container %s', str(container_name))
+                            configlet['containers'].append(container_name)
+            facts['configlets'].append(configlet)
+    else:
+        MODULE_LOGGER.error('No configlet found on CVP')
+    MODULE_LOGGER.info('All configlets facts collected')
     return facts
 
 
@@ -363,16 +444,16 @@ def facts_builder(module):
         MODULE_LOGGER.info('** Collecting devices facts ...')
         facts = facts_devices(module=module, facts=facts)
 
-    # Extract configlet information
-    if 'all' in module.params['facts'] or 'configlets' in module.params['facts']:
-        MODULE_LOGGER.info('** Collecting configlets facts ...')
-        facts = facts_configlets(module=module, facts=facts)
-
     # Extract containers information
     if 'all' in module.params['facts'] or 'containers' in module.params['facts']:
         MODULE_LOGGER.info('** Collecting containers facts ...')
         facts = facts_containers(module=module, facts=facts)
 
+    # Extract configlet information
+    if 'all' in module.params['facts'] or 'configlets' in module.params['facts']:
+        MODULE_LOGGER.info('** Collecting configlets facts ...')
+        facts = facts_configlets(module=module, facts=facts)
+
     # Extract tasks information
     if 'all' in module.params['facts'] or 'tasks' in module.params['facts']:
         MODULE_LOGGER.info('** Collecting tasks facts ...')