Fix(eos_designs): Fix schema validation of dynamic keys (#4474)

ansible_collections/arista/avd/plugins/action/eos_designs_facts.py

@@ -144,15 +144,6 @@ def create_avd_switch_facts_instances(self, fabric_hosts: list, hostvars: object
             # Fetch all templated Ansible vars for this host
             host_hostvars = dict(hostvars.get(host))
 
-            # Initialize SharedUtils class to be passed to EosDesignsFacts below.
-            shared_utils = SharedUtils(hostvars=host_hostvars, templar=self.templar, schema=avdschematools.avdschema)
-
-            # Insert dynamic keys into the input data if not set.
-            # These keys are required by the schema, but the default values are set inside shared_utils.
-            host_hostvars.setdefault("node_type_keys", shared_utils.node_type_keys)
-            host_hostvars.setdefault("connected_endpoints_keys", shared_utils.connected_endpoints_keys)
-            host_hostvars.setdefault("network_services_keys", shared_utils.network_services_keys)
-
             # Set correct hostname in schema tools and perform conversion and validation
             avdschematools.hostname = host
             host_result = avdschematools.convert_and_validate_data(host_hostvars, return_counters=True)
@@ -168,6 +159,9 @@ def create_avd_switch_facts_instances(self, fabric_hosts: list, hostvars: object
             # This is used to access EosDesignsFacts objects of other switches during rendering of one switch.
             host_hostvars["avd_switch_facts"] = avd_switch_facts
 
+            # Initialize SharedUtils class to be passed to EosDesignsFacts below.
+            shared_utils = SharedUtils(hostvars=host_hostvars, templar=self.templar, schema=avdschematools.avdschema)
+
             # Create an instance of EosDesignsFacts and insert into common avd_switch_facts dict
             avd_switch_facts[host] = {"switch": EosDesignsFacts(hostvars=host_hostvars, shared_utils=shared_utils)}
 

python-avd/pyavd/_eos_designs/schema/schema_fragments/connected_endpoints.schema.yml

@@ -12,42 +12,3 @@ dynamic_keys:
     display_name: Connected Endpoints
     documentation_options:
       table: connected-endpoints
-    default:
-      # NOTE: there is a static list of default endpoint keys in the
-      # fabric connected endpoints documentation templates.
-      - key: servers
-        type: server
-        description: Server
-      - key: firewalls
-        type: firewall
-        description: Firewall
-      - key: routers
-        type: router
-        description: Router
-      - key: load_balancers
-        type: load_balancer
-        description: Load Balancer
-      - key: storage_arrays
-        type: storage_array
-        description: Storage Array
-      - key: cpes
-        type: cpe
-        description: CPE
-      - key: workstations
-        type: workstation
-        description: Workstation
-      - key: access_points
-        type: access_point
-        description: Access Point
-      - key: phones
-        type: phone
-        description: Phone
-      - key: printers
-        type: printer
-        description: Printer
-      - key: cameras
-        type: camera
-        description: Camera
-      - key: generic_devices
-        type: generic_device
-        description: Generic Device

python-avd/pyavd/_eos_designs/schema/schema_fragments/custom_node_type.schema.yml

@@ -0,0 +1,13 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+# yaml-language-server: $schema=../../../_schema/avd_meta_schema.json
+# Line above is used by RedHat's YAML Schema vscode extension
+# Use Ctrl + Space to get suggestions for every field. Autocomplete will pop up after typing 2 letters.
+type: dict
+dynamic_keys:
+  "custom_node_type_keys.key":
+    $ref: "eos_designs#/$defs/node_type"
+    type: dict
+    documentation_options:
+      hide_keys: true

python-avd/pyavd/_eos_designs/schema/schema_fragments/defs_connected_endpoints.schema.yml

@@ -12,7 +12,6 @@ $defs:
     description: |
       This should be applied to group_vars or host_vars where endpoints are connecting.
       `connected_endpoints_keys.key` is one of the keys under "connected_endpoints_keys".
-      The default keys are `servers`, `firewalls`, `routers`, `load_balancers`, and `storage_arrays`.
     items:
       type: dict
       keys:

python-avd/pyavd/_eos_designs/schema/schema_fragments/type.schema.yml

@@ -10,7 +10,9 @@ keys:
     documentation_options:
       table: type-setting
     type: str
-    dynamic_valid_values: "node_type_keys.type"
+    dynamic_valid_values:
+      - custom_node_type_keys.type
+      - node_type_keys.type
     description: |
       The `type:` variable needs to be defined for each device in the fabric.
       This is leveraged to load the appropriate template to generate the configuration.

python-avd/pyavd/_eos_designs/structured_config/__init__.py

@@ -78,28 +78,19 @@ def get_structured_config(
     Returns:
         The structured_config as a dict
     """
-    structured_config = {}
-    module_vars = ChainMap(
-        structured_config,
-        vars,
-    )
-
-    # Initialize SharedUtils class to be passed to each python_module below.
-    shared_utils = SharedUtils(hostvars=module_vars, templar=templar, schema=input_schema_tools.avdschema)
-
-    # Insert dynamic keys into the input data if not set.
-    # These keys are required by the schema, but the default values are set inside shared_utils.
-    vars.setdefault("node_type_keys", shared_utils.node_type_keys)
-    vars.setdefault("connected_endpoints_keys", shared_utils.connected_endpoints_keys)
-    vars.setdefault("network_services_keys", shared_utils.network_services_keys)
-
     # Validate input data
     if validate:
         result.update(input_schema_tools.convert_and_validate_data(vars))
         if result.get("failed"):
             # Input data validation failed so return empty dict. Calling function should check result.get("failed").
             return {}
 
+    structured_config = {}
+    module_vars = ChainMap(structured_config, vars)
+
+    # Initialize SharedUtils class to be passed to each python_module below.
+    shared_utils = SharedUtils(hostvars=module_vars, templar=templar, schema=input_schema_tools.avdschema)
+
     for cls in AVD_STRUCTURED_CONFIG_CLASSES:
         eos_designs_module: AvdFacts = cls(module_vars, shared_utils)
         results = eos_designs_module.render()

python-avd/pyavd/_schema/avd_meta_schema.json

@@ -376,8 +376,11 @@
             "description": "Key is required"
         },
         "dynamic_valid_values": {
-            "type": "string",
-            "description": "Path to variable under the parent dictionary containing valid values.\nVariable path use dot-notation and variable path must be relative to the parent dictionary.\nIf an element of the variable path is a list, every list item will be unpacked.\nNote that this is building the schema from values in the _data_ being validated!"
+            "type": "array",
+            "items": {
+                "type": "string",
+                "description": "Path to variable under the parent dictionary containing valid values.\nVariable path use dot-notation and variable path must be relative to the parent dictionary.\nIf an element of the variable path is a list, every list item will be unpacked.\nNote that this is building the schema from values in the _data_ being validated!"
+            }
         },
         "$ref": {
             "type": "string",

python-avd/pyavd/_schema/avddataconverter.py

@@ -8,6 +8,8 @@
 from pyavd._errors import AvdDeprecationWarning
 from pyavd._utils import get_all
 
+from .utils import get_instance_with_defaults
+
 SCHEMA_TO_PY_TYPE_MAP = {
     "str": str,
     "int": int,
@@ -92,7 +94,8 @@ def convert_dynamic_keys(self, dynamic_keys: dict, data: dict, schema: dict, pat
         # Resolve "keys" from schema "dynamic_keys" by looking for the dynamic key in data.
         keys = {}
         for dynamic_key, childschema in dynamic_keys.items():
-            resolved_keys = get_all(data, dynamic_key)
+            data_with_defaults = get_instance_with_defaults(data, dynamic_key, schema)
+            resolved_keys = get_all(data_with_defaults, dynamic_key)
             for resolved_key in resolved_keys:
                 keys.setdefault(resolved_key, childschema)
 

python-avd/pyavd/_schema/avdvalidator.py

@@ -9,6 +9,8 @@
 from pyavd._errors import AvdValidationError
 from pyavd._utils import get_all, get_all_with_path, get_indices_of_duplicate_items
 
+from .utils import get_instance_with_defaults
+
 
 class AvdValidator:
     def __init__(self, schema: dict) -> None:
@@ -109,7 +111,8 @@ def keys_validator(self, keys: dict, instance: dict, schema: dict, path: list[st
         schema_dynamic_keys = schema.get("dynamic_keys", {})
         dynamic_keys = {}
         for dynamic_key, childschema in schema_dynamic_keys.items():
-            resolved_keys = get_all(instance, dynamic_key)
+            instance_with_defaults = get_instance_with_defaults(instance, dynamic_key, schema)
+            resolved_keys = get_all(instance_with_defaults, dynamic_key)
             for resolved_key in resolved_keys:
                 dynamic_keys.setdefault(resolved_key, childschema)
 
@@ -124,7 +127,8 @@ def keys_validator(self, keys: dict, instance: dict, schema: dict, path: list[st
 
         # Run over child keys and check for required and update child schema with dynamic valid values before
         # descending into validation of child schema.
-        for key, childschema in all_keys.items():
+        for key in all_keys:
+            childschema = all_keys[key].copy()
             if instance.get(key) is None:
                 # Validation of "required" on child keys
                 if childschema.get("required"):
@@ -135,7 +139,9 @@ def keys_validator(self, keys: dict, instance: dict, schema: dict, path: list[st
 
             # Expand "dynamic_valid_values" in child schema and add to "valid_values"
             if "dynamic_valid_values" in childschema:
-                childschema.setdefault("valid_values", []).extend(get_all(instance, childschema["dynamic_valid_values"]))
+                for dynamic_valid_value in childschema["dynamic_valid_values"]:
+                    instance_with_defaults = get_instance_with_defaults(instance, dynamic_valid_value, schema)
+                    childschema.setdefault("valid_values", []).extend(get_all(instance_with_defaults, dynamic_valid_value))
 
             # Perform regular validation of the child schema.
             yield from self.validate(