Enhance inventory with groups from CMDB relations (#143)

Enhance inventory with groups from CMDB relations

SUMMARY
Extends inventory plugin module with groups extracted from CMDB relations.
Addresses issue #108.
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
Changes to now, added relations module utilities.
ADDITIONAL INFORMATION
To enable this extension, add option enhanced: true to the inventory file.
Unlike the implementation from the old, deprecated repository, which required installation of an update set, this implementation obtains all the required data through ServiceNow REST API.

Reviewed-by: Abhijeet Kasurde <None>
Reviewed-by: None <None>

changelogs/fragments/enhanced-inventory.yaml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+- now - Enhance inventory with additional groups from CMDB relations (https://github.com/ansible-collections/servicenow.itsm/issues/108).

plugins/inventory/now.py

@@ -21,6 +21,12 @@
 from ..module_utils.errors import ServiceNowError
 from ..module_utils.query import parse_query, serialize_query
 from ..module_utils.table import TableClient
+from ..module_utils.relations import (
+    REL_FIELDS,
+    REL_QUERY,
+    REL_TABLE,
+    enhance_records_with_rel_groups,
+)
 
 
 DOCUMENTATION = r"""
@@ -30,6 +36,7 @@
   - Manca Bizjak (@mancabizjak)
   - Miha Dolinar (@mdolin)
   - Tadej Borovsak (@tadeboro)
+  - Uros Pascinski (@uscinski)
 short_description: Inventory source for ServiceNow table records.
 description:
   - Builds inventory from ServiceNow table records.
@@ -105,6 +112,13 @@
       - List of I(table) columns to be included as hostvars.
     type: list
     default: [name, host_name, fqdn, ip_address]
+  enhanced:
+    description:
+      - Enable enhanced inventory which provides relationship information from CMDB.
+      - Mutually exclusive with deprecated options I(named_groups) and I(group_by).
+    type: bool
+    default: false
+    version_added: 1.3.0
   ansible_host_source:
     description:
       - Host variable to use as I(ansible_host) when generating inventory hosts.
@@ -291,6 +305,29 @@
 #  |  |  |--{cpu_type = Intel}
 #  |  |  |--{name = SAP-SD-02}
 
+# Similar to the example above, but use enhanced groups with relationship information instead.
+plugin: servicenow.itsm.now
+enhanced: true
+strict: true
+inventory_hostname_source: asset_tag
+columns:
+  - name
+  - classification
+  - cpu_type
+  - cost
+compose:
+    cost: cost ~ " " ~ cost_cc
+    ansible_host: fqdn
+
+# `ansible-inventory -i inventory.now.yaml --graph --vars` output:
+# @all:
+# |--@Blackberry_Depends_on:
+# |  |--P1000201
+# |  |  |--{ansible_host = my.server.com}
+# |  |  |--{classification = Production}
+# |  |  |--{cost = 2,160 USD}
+# |  |  |--{cpu_type = Intel}
+# |  |  |--{name = INSIGHT-NY-03}
 
 # NOTE: All examples from here on are deprecated and should not be used when writing new
 # inventory sources.
@@ -392,14 +429,17 @@ def construct_sysparm_query(query):
     return serialize_query(parsed)
 
 
-def fetch_records(table_client, table, query):
+def fetch_records(table_client, table, query, fields=None):
     snow_query = dict(
         # Make references and choice fields human-readable
         sysparm_display_value=True,
     )
     if query:
         snow_query["sysparm_query"] = construct_sysparm_query(query)
 
+    if fields:
+        snow_query["sysparm_fields"] = ",".join(fields)
+
     return table_client.list_records(table, snow_query)
 
 
@@ -537,6 +577,7 @@ def fill_constructed(
         groups,
         keyed_groups,
         strict,
+        enhanced,
     ):
         for record in records:
             host = self.add_host(record, host_source, name_source)
@@ -545,6 +586,14 @@ def fill_constructed(
                 self._set_composite_vars(compose, record, host, strict)
                 self._add_host_to_composed_groups(groups, record, host, strict)
                 self._add_host_to_keyed_groups(keyed_groups, record, host, strict)
+                if enhanced:
+                    self.fill_enhanced_auto_groups(record, host)
+
+    def fill_enhanced_auto_groups(self, record, host):
+        for rel_group in record["relationship_groups"]:
+            rel_group = to_safe_group_name(rel_group)
+            self.inventory.add_group(rel_group)
+            self.inventory.add_child(rel_group, host)
 
     def _merge_instance_config(self, instance_config, instance_env):
         # Pulls the values from the environment, and if necessary, overrides
@@ -605,6 +654,15 @@ def parse(self, inventory, loader, path, cache=True):
             client = Client(**self._get_instance())
         except ServiceNowError as e:
             raise AnsibleParserError(e)
+
+        enhanced = self.get_option("enhanced")
+
+        if enhanced and (named_groups or group_by):
+            raise AnsibleParserError(
+                "Option 'enhanced' is incompatible with options 'named_groups' or "
+                "'group_by'."
+            )
+
         table_client = TableClient(client)
 
         table = self.get_option("table")
@@ -626,17 +684,25 @@ def parse(self, inventory, loader, path, cache=True):
             )
             return
 
+        query = self.get_option("query")
+
         # TODO: Insert caching here once we remove deprecated functionality
-        records = fetch_records(
-            table_client, self.get_option("table"), self.get_option("query")
-        )
+        records = fetch_records(table_client, table, query)
+
+        if enhanced:
+            rel_records = fetch_records(
+                table_client, REL_TABLE, REL_QUERY, fields=REL_FIELDS
+            )
+            enhance_records_with_rel_groups(records, rel_records)
+
         self.fill_constructed(
             records,
-            self.get_option("columns"),
-            self.get_option("ansible_host_source"),
-            self.get_option("inventory_hostname_source"),
+            columns,
+            host_source,
+            name_source,
             self.get_option("compose"),
             self.get_option("groups"),
             self.get_option("keyed_groups"),
             self.get_option("strict"),
+            enhanced,
         )

plugins/module_utils/relations.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# Copyright: (c) 2022, XLAB Steampunk <[email protected]>
+#
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import re
+
+
+REL_TABLE = "cmdb_rel_ci"
+# sysparm_fields to be used when querying REL_TABLE. Uses dot-walking
+# notation to extract fields from linked tables in a single REST API call.
+# https://docs.servicenow.com/bundle/rome-application-development/page/integrate/inbound-rest/concept/c_RESTAPI.html#d1168970e439
+REL_FIELDS = set(
+    (
+        "sys_id",
+        "type.name",
+        "parent.sys_id",
+        "parent.name",
+        "parent.sys_class_name",
+        "child.sys_id",
+        "child.name",
+        "child.sys_class_name",
+    )
+)
+
+# Similar as above but for sysparm_query
+REL_QUERY = None
+
+
+def _extend_records_with_groups(records, groups):
+    for record in records:
+        sys_id = record.get("sys_id")
+        sys_id_groups = groups.get(sys_id, set())
+        record["relationship_groups"] = sys_id_groups
+
+    return records
+
+
+def _extract_ci_rel_type(type_name):
+    # type_name is of form "Parent description::Child description".
+    # Return the value of form (Parent_description, Child_description).
+    type_name = type_name or "__"
+    type_name = re.sub(r"\s|:", "_", type_name)
+    ci_rel_type = tuple(type_name.split("__"))
+
+    return ci_rel_type
+
+
+def _extract_parent_relation(rel_record):
+    sys_id = rel_record.get("parent.sys_id", "")
+    ci_name = rel_record.get("child.name", "")
+    ci_class = rel_record.get("child.sys_class_name", "")
+    type_name = rel_record.get("type.name", "")
+    ci_rel_type = _extract_ci_rel_type(type_name)[1]
+
+    return sys_id, ci_name, ci_class, ci_rel_type
+
+
+def _extract_child_relation(rel_record):
+    sys_id = rel_record.get("child.sys_id", "")
+    ci_name = rel_record.get("parent.name", "")
+    ci_class = rel_record.get("parent.sys_class_name", "")
+    type_name = rel_record.get("type.name", "")
+    ci_rel_type = _extract_ci_rel_type(type_name)[0]
+
+    return sys_id, ci_name, ci_class, ci_rel_type
+
+
+def _relations_to_groups(rel_records):
+    groups = dict()
+
+    extract_relation = dict(
+        parent=_extract_parent_relation, child=_extract_child_relation
+    )
+
+    for rel_record in rel_records or list():
+        for target in ("child", "parent"):
+            t_extr_rel = extract_relation[target]
+            sys_id, ci_name, ci_class, ci_rel_type = t_extr_rel(rel_record)
+
+            if sys_id and ci_name and ci_rel_type and ci_class:
+                rel_group = "{0}_{1}".format(ci_name, ci_rel_type)
+
+                items = groups.setdefault(sys_id, set())
+                items.add(rel_group)
+
+    return groups
+
+
+def enhance_records_with_rel_groups(records, rel_records):
+    groups = _relations_to_groups(rel_records)
+    records = _extend_records_with_groups(records, groups)
+
+    return records